diff options
440 files changed, 15793 insertions, 6867 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b915cfabc7..691582239e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -82,10 +82,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [bionic]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_win64.sh" @@ -136,7 +136,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[no depends, sanitizers: fuzzer,address,undefined] [focal]' + name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -160,10 +160,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" task: - name: 'macOS 10.14 [gui, no tests] [bionic]' + name: 'macOS 10.14 [gui, no tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_mac.sh" diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4967e675f6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# This is the top-most EditorConfig file. +root = true + +# For all files. +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +# Source code files +[*.{h,cpp,py,sh}] +indent_size = 4 + +# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +[*.yml] +indent_size = 2 + +# Makefiles +[{*.am,Makefile.*.include}] +indent_style = tab + +# Autoconf scripts +[configure.ac] +indent_size = 2 diff --git a/.fuzzbuzz.yml b/.fuzzbuzz.yml index be9a1cd4e1..e40b4df165 100644 --- a/.fuzzbuzz.yml +++ b/.fuzzbuzz.yml @@ -5,7 +5,7 @@ environment: - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1 setup: - sudo apt-get update - - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libqt5core5a libqt5dbus5 libqt5gui5 libtool llvm llvm-dev llvm-runtime pkg-config qttools5-dev qttools5-dev-tools software-properties-common + - sudo apt-get install -y autoconf bsdmainutils clang git libboost-system-dev libboost-filesystem-dev libboost-test-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libqt5core5a libqt5dbus5 libqt5gui5 libtool llvm llvm-dev llvm-runtime pkg-config qttools5-dev qttools5-dev-tools software-properties-common - ./autogen.sh - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined --enable-danger-fuzz-link-all - make diff --git a/.gitignore b/.gitignore index 810ef5db6b..c0b5661464 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,9 @@ src/bitcoin-cli src/bitcoin-gui src/bitcoin-node src/bitcoin-tx +src/bitcoin-util src/bitcoin-wallet -src/test/fuzz/* -!src/test/fuzz/*.* +src/test/fuzz/fuzz src/test/test_bitcoin src/qt/test/test_bitcoin-qt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e11474382..ae2379fbd5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -197,7 +197,7 @@ Note: Code review is a burdensome but important part of the development process, If your pull request contains fixup commits (commits that change the same line of code repeatedly) or too fine-grained commits, you may be asked to [squash](https://git-scm.com/docs/git-rebase#_interactive_mode) your commits -before it will be merged. The basic squashing workflow is shown below. +before it will be reviewed. The basic squashing workflow is shown below. git checkout your_branch_name git rebase -i HEAD~n diff --git a/Makefile.am b/Makefile.am index b4a55907df..66be768277 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4,7 +4,7 @@ # Pattern rule to print variables, e.g. make print-top_srcdir print-%: - @echo $* = $($*) + @echo '$*' = '$($*)' ACLOCAL_AMFLAGS = -I build-aux/m4 SUBDIRS = src @@ -25,6 +25,7 @@ BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) +BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT) BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) BITCOIN_NODE_BIN=$(top_builddir)/src/$(BITCOIN_MP_NODE_NAME)$(EXEEXT) BITCOIN_GUI_BIN=$(top_builddir)/src/$(BITCOIN_MP_GUI_NAME)$(EXEEXT) @@ -45,6 +46,9 @@ OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed DIST_CONTRIB = \ + $(top_srcdir)/test/sanitizer_suppressions/lsan \ + $(top_srcdir)/test/sanitizer_suppressions/tsan \ + $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ $(top_srcdir)/contrib/linearize/linearize-hashes.py @@ -81,6 +85,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_UTIL_BIN) $(top_builddir)/release @test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \ echo error: could not build $@ @echo built $@ @@ -139,7 +144,7 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) - $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) $(OSX_DMG): $(OSX_TEMP_ISO) $(DMG) dmg "$<" "$@" @@ -177,6 +182,9 @@ $(BITCOIN_CLI_BIN): FORCE $(BITCOIN_TX_BIN): FORCE $(MAKE) -C src $(@F) +$(BITCOIN_UTIL_BIN): FORCE + $(MAKE) -C src $(@F) + $(BITCOIN_WALLET_BIN): FORCE $(MAKE) -C src $(@F) @@ -3,6 +3,11 @@ Bitcoin Core integration/staging tree https://bitcoincore.org +For an immediately usable, binary version of the Bitcoin Core software, see +https://bitcoincore.org/en/download/. + +Further information about Bitcoin Core is available in the [doc folder](/doc). + What is Bitcoin? ---------------- @@ -12,9 +17,7 @@ with no central authority: managing transactions and issuing money are carried out collectively by the network. Bitcoin Core is the name of open source software which enables the use of this currency. -For more information, as well as an immediately usable, binary version of -the Bitcoin Core software, see https://bitcoincore.org/en/download/, or read the -[original whitepaper](https://bitcoincore.org/bitcoin.pdf). +For more information read the original Bitcoin whitepaper. License ------- @@ -53,7 +56,7 @@ submit new unit tests for old code. Unit tests can be compiled and run and extending unit tests can be found in [/src/test/README.md](/src/test/README.md). There are also [regression and integration tests](/test), written -in Python, that are run automatically on the build server. +in Python. These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py` The CI (Continuous Integration) systems make sure that every pull request is built for Windows, Linux, and macOS, @@ -77,5 +80,3 @@ Translations are periodically pulled from Transifex and merged into the git repo **Important**: We do not accept translation changes as GitHub pull requests because the next pull from Transifex would automatically overwrite them again. - -Translators should also subscribe to the [mailing list](https://groups.google.com/forum/#!forum/bitcoin-translators). diff --git a/build-aux/m4/ax_boost_process.m4 b/build-aux/m4/ax_boost_process.m4 deleted file mode 100644 index 5d20e67464..0000000000 --- a/build-aux/m4/ax_boost_process.m4 +++ /dev/null @@ -1,121 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_process.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_PROCESS -# -# DESCRIPTION -# -# Test for Process library from the Boost C++ libraries. The macro -# requires a preceding call to AX_BOOST_BASE. Further documentation is -# available at <http://randspringer.de/boost/index.html>. -# -# This macro calls: -# -# AC_SUBST(BOOST_PROCESS_LIB) -# -# And sets: -# -# HAVE_BOOST_PROCESS -# -# LICENSE -# -# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> -# Copyright (c) 2008 Michael Tindal -# Copyright (c) 2008 Daniel Casimiro <dan.casimiro@gmail.com> -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 2 - -AC_DEFUN([AX_BOOST_PROCESS], -[ - AC_ARG_WITH([boost-process], - AS_HELP_STRING([--with-boost-process@<:@=special-lib@:>@], - [use the Process library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-process=boost_process-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost_process="no" - elif test "$withval" = "yes"; then - want_boost_process="yes" - ax_boost_user_process_lib="" - else - want_boost_process="yes" - ax_boost_user_process_lib="$withval" - fi - ], - [want_boost_process="yes"] - ) - - if test "x$want_boost_process" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([AC_CANONICAL_BUILD]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::Process library is available, - ax_cv_boost_process, - [AC_LANG_PUSH([C++]) - CXXFLAGS_SAVE=$CXXFLAGS - CXXFLAGS= - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/process.hpp>]], - [[boost::process::child* child = new boost::process::child; delete child;]])], - ax_cv_boost_process=yes, ax_cv_boost_process=no) - CXXFLAGS=$CXXFLAGS_SAVE - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_process" = "xyes"; then - AC_SUBST(BOOST_CPPFLAGS) - - AC_DEFINE(HAVE_BOOST_PROCESS,,[define if the Boost::Process library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - LDFLAGS_SAVE=$LDFLAGS - if test "x$ax_boost_user_process_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_process* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], - [link_process="no"]) - done - if test "x$link_process" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_process* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], - [link_process="no"]) - done - fi - - else - for ax_lib in $ax_boost_user_process_lib boost_process-$ax_boost_user_process_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], - [link_process="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Process library!) - fi - if test "x$link_process" = "xno"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_boost_thread.m4 b/build-aux/m4/ax_boost_thread.m4 deleted file mode 100644 index 75e80e6e75..0000000000 --- a/build-aux/m4/ax_boost_thread.m4 +++ /dev/null @@ -1,187 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_thread.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_THREAD -# -# DESCRIPTION -# -# Test for Thread library from the Boost C++ libraries. The macro requires -# a preceding call to AX_BOOST_BASE. Further documentation is available at -# <http://randspringer.de/boost/index.html>. -# -# This macro calls: -# -# AC_SUBST(BOOST_THREAD_LIB) -# -# And sets: -# -# HAVE_BOOST_THREAD -# -# LICENSE -# -# Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de> -# Copyright (c) 2009 Michael Tindal -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 33 - -AC_DEFUN([AX_BOOST_THREAD], -[ - AC_ARG_WITH([boost-thread], - AS_HELP_STRING([--with-boost-thread@<:@=special-lib@:>@], - [use the Thread library from boost - - it is possible to specify a certain library for the linker - e.g. --with-boost-thread=boost_thread-gcc-mt ]), - [ - if test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_thread_lib="" - else - want_boost="yes" - ax_boost_user_thread_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([AC_CANONICAL_BUILD]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::Thread library is available, - ax_cv_boost_thread, - [AC_LANG_PUSH([C++]) - CXXFLAGS_SAVE=$CXXFLAGS - - case "x$host_os" in - xsolaris ) - CXXFLAGS="-pthreads $CXXFLAGS" - break; - ;; - xmingw32 ) - CXXFLAGS="-mthreads $CXXFLAGS" - break; - ;; - *android* ) - break; - ;; - * ) - CXXFLAGS="-pthread $CXXFLAGS" - break; - ;; - esac - - AC_COMPILE_IFELSE([ - AC_LANG_PROGRAM( - [[@%:@include <boost/thread/thread.hpp>]], - [[boost::thread_group thrds; - return 0;]])], - ax_cv_boost_thread=yes, ax_cv_boost_thread=no) - CXXFLAGS=$CXXFLAGS_SAVE - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_thread" = "xyes"; then - case "x$host_os" in - xsolaris ) - BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" - break; - ;; - xmingw32 ) - BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" - break; - ;; - *android* ) - break; - ;; - * ) - BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" - break; - ;; - esac - - AC_SUBST(BOOST_CPPFLAGS) - - AC_DEFINE(HAVE_BOOST_THREAD,, - [define if the Boost::Thread library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - LDFLAGS_SAVE=$LDFLAGS - case "x$host_os" in - *bsd* ) - LDFLAGS="-pthread $LDFLAGS" - break; - ;; - esac - if test "x$ax_boost_user_thread_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_thread* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'`; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [link_thread="yes"; break], - [link_thread="no"]) - done - if test "x$link_thread" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_thread* 2>/dev/null | sed 's,.*/,,' | sed 's,\..*,,'`; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [link_thread="yes"; break], - [link_thread="no"]) - done - fi - - else - for ax_lib in $ax_boost_user_thread_lib boost_thread-$ax_boost_user_thread_lib; do - AC_CHECK_LIB($ax_lib, exit, - [link_thread="yes"; break], - [link_thread="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Thread library!) - fi - if test "x$link_thread" = "xno"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - else - BOOST_THREAD_LIB="-l$ax_lib" - case "x$host_os" in - *bsd* ) - BOOST_LDFLAGS="-pthread $BOOST_LDFLAGS" - break; - ;; - xsolaris ) - BOOST_THREAD_LIB="$BOOST_THREAD_LIB -lpthread" - break; - ;; - xmingw32 ) - break; - ;; - *android* ) - break; - ;; - * ) - BOOST_THREAD_LIB="$BOOST_THREAD_LIB -lpthread" - break; - ;; - esac - AC_SUBST(BOOST_THREAD_LIB) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_gcc_func_attribute.m4 b/build-aux/m4/ax_gcc_func_attribute.m4 deleted file mode 100644 index c788ca9bd4..0000000000 --- a/build-aux/m4/ax_gcc_func_attribute.m4 +++ /dev/null @@ -1,223 +0,0 @@ -# =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_gcc_func_attribute.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_GCC_FUNC_ATTRIBUTE(ATTRIBUTE) -# -# DESCRIPTION -# -# This macro checks if the compiler supports one of GCC's function -# attributes; many other compilers also provide function attributes with -# the same syntax. Compiler warnings are used to detect supported -# attributes as unsupported ones are ignored by default so quieting -# warnings when using this macro will yield false positives. -# -# The ATTRIBUTE parameter holds the name of the attribute to be checked. -# -# If ATTRIBUTE is supported define HAVE_FUNC_ATTRIBUTE_<ATTRIBUTE>. -# -# The macro caches its result in the ax_cv_have_func_attribute_<attribute> -# variable. -# -# The macro currently supports the following function attributes: -# -# alias -# aligned -# alloc_size -# always_inline -# artificial -# cold -# const -# constructor -# constructor_priority for constructor attribute with priority -# deprecated -# destructor -# dllexport -# dllimport -# error -# externally_visible -# flatten -# format -# format_arg -# gnu_inline -# hot -# ifunc -# leaf -# malloc -# noclone -# noinline -# nonnull -# noreturn -# nothrow -# optimize -# pure -# unused -# used -# visibility -# warning -# warn_unused_result -# weak -# weakref -# -# Unsuppored function attributes will be tested with a prototype returning -# an int and not accepting any arguments and the result of the check might -# be wrong or meaningless so use with care. -# -# LICENSE -# -# Copyright (c) 2013 Gabriele Svelto <gabriele.svelto@gmail.com> -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 3 - -AC_DEFUN([AX_GCC_FUNC_ATTRIBUTE], [ - AS_VAR_PUSHDEF([ac_var], [ax_cv_have_func_attribute_$1]) - - AC_CACHE_CHECK([for __attribute__(($1))], [ac_var], [ - AC_LINK_IFELSE([AC_LANG_PROGRAM([ - m4_case([$1], - [alias], [ - int foo( void ) { return 0; } - int bar( void ) __attribute__(($1("foo"))); - ], - [aligned], [ - int foo( void ) __attribute__(($1(32))); - ], - [alloc_size], [ - void *foo(int a) __attribute__(($1(1))); - ], - [always_inline], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [artificial], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [cold], [ - int foo( void ) __attribute__(($1)); - ], - [const], [ - int foo( void ) __attribute__(($1)); - ], - [constructor_priority], [ - int foo( void ) __attribute__((__constructor__(65535/2))); - ], - [constructor], [ - int foo( void ) __attribute__(($1)); - ], - [deprecated], [ - int foo( void ) __attribute__(($1(""))); - ], - [destructor], [ - int foo( void ) __attribute__(($1)); - ], - [dllexport], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [dllimport], [ - int foo( void ) __attribute__(($1)); - ], - [error], [ - int foo( void ) __attribute__(($1(""))); - ], - [externally_visible], [ - int foo( void ) __attribute__(($1)); - ], - [flatten], [ - int foo( void ) __attribute__(($1)); - ], - [format], [ - int foo(const char *p, ...) __attribute__(($1(printf, 1, 2))); - ], - [format_arg], [ - char *foo(const char *p) __attribute__(($1(1))); - ], - [gnu_inline], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [hot], [ - int foo( void ) __attribute__(($1)); - ], - [ifunc], [ - int my_foo( void ) { return 0; } - static int (*resolve_foo(void))(void) { return my_foo; } - int foo( void ) __attribute__(($1("resolve_foo"))); - ], - [leaf], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [malloc], [ - void *foo( void ) __attribute__(($1)); - ], - [noclone], [ - int foo( void ) __attribute__(($1)); - ], - [noinline], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [nonnull], [ - int foo(char *p) __attribute__(($1(1))); - ], - [noreturn], [ - void foo( void ) __attribute__(($1)); - ], - [nothrow], [ - int foo( void ) __attribute__(($1)); - ], - [optimize], [ - __attribute__(($1(3))) int foo( void ) { return 0; } - ], - [pure], [ - int foo( void ) __attribute__(($1)); - ], - [unused], [ - int foo( void ) __attribute__(($1)); - ], - [used], [ - int foo( void ) __attribute__(($1)); - ], - [visibility], [ - int foo_def( void ) __attribute__(($1("default"))); - int foo_hid( void ) __attribute__(($1("hidden"))); - int foo_int( void ) __attribute__(($1("internal"))); - int foo_pro( void ) __attribute__(($1("protected"))); - ], - [warning], [ - int foo( void ) __attribute__(($1(""))); - ], - [warn_unused_result], [ - int foo( void ) __attribute__(($1)); - ], - [weak], [ - int foo( void ) __attribute__(($1)); - ], - [weakref], [ - static int foo( void ) { return 0; } - static int bar( void ) __attribute__(($1("foo"))); - ], - [ - m4_warn([syntax], [Unsupported attribute $1, the test may fail]) - int foo( void ) __attribute__(($1)); - ] - )], []) - ], - dnl GCC doesn't exit with an error if an unknown attribute is - dnl provided but only outputs a warning, so accept the attribute - dnl only if no warning were issued. - [AS_IF([test -s conftest.err], - [AS_VAR_SET([ac_var], [no])], - [AS_VAR_SET([ac_var], [yes])])], - [AS_VAR_SET([ac_var], [no])]) - ]) - - AS_IF([test yes = AS_VAR_GET([ac_var])], - [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_FUNC_ATTRIBUTE_$1), 1, - [Define to 1 if the system has the `$1' function attribute])], []) - - AS_VAR_POPDEF([ac_var]) -]) diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 75c43f9a92..5201a8cc7c 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -14,6 +14,9 @@ m4_define([_CHECK_ATOMIC_testbody], [[ #include <cstdint> int main() { + std::atomic<bool> lock{true}; + std::atomic_exchange(&lock, false); + std::atomic<int64_t> a{}; int64_t v = 5; diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 53aead38b5..40a30b9749 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -50,15 +50,12 @@ /* define if the Boost::Filesystem library is available */ #define HAVE_BOOST_FILESYSTEM /**/ -/* define if the Boost::Process library is available */ -#define HAVE_BOOST_PROCESS /**/ +/* define if external signer support is enabled (requires Boost::Process) */ +#define ENABLE_EXTERNAL_SIGNER /**/ /* define if the Boost::System library is available */ #define HAVE_BOOST_SYSTEM /**/ -/* define if the Boost::Thread library is available */ -#define HAVE_BOOST_THREAD /**/ - /* define if the Boost::Unit_Test_Framework library is available */ #define HAVE_BOOST_UNIT_TEST_FRAMEWORK /**/ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 9c589bccbc..a3487c7a0d 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -96,7 +96,7 @@ <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> - <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 6a3c9f1dc1..490ce8b1ce 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -104,6 +104,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondesc.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondescdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionfilterproxy.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionoverviewwidget.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionrecord.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiontablemodel.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionview.cpp" /> diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index dfd3929c44..42b9a5d16f 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -8,7 +8,6 @@ "boost-process", "boost-signals2", "boost-test", - "boost-thread", "sqlite3", "double-conversion", { diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index a0b579de1e..343b82a1ad 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -15,6 +15,7 @@ ${CI_RETRY_EXE} pip3 install codespell==2.0.0 ${CI_RETRY_EXE} pip3 install flake8==3.8.3 ${CI_RETRY_EXE} pip3 install yq ${CI_RETRY_EXE} pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install vulture==2.3 SHELLCHECK_VERSION=v0.7.1 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index de89a09d6a..e38cfe8eef 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -21,7 +21,6 @@ test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/check-rpc-mappings.py . test/lint/lint-all.sh if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index b0de2ec0bb..6da011d19b 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,9 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index f682486088..e47119e6fa 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 export GOAL="install" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index a32de4a6b5..ebb5a1cabe 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,11 +8,11 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++ --with-boost-process" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index e06a40eb23..2cf672b91e 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 522a5d9fc2..c5692d786a 100644 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -13,3 +13,4 @@ export DEP_OPTS="MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--with-boost-process" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" +export RUN_SECURITY_TESTS="true" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 567145fe47..4c42605e9a 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -11,9 +11,9 @@ export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash -export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index f0c153158b..e079a7057c 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index affaaaa1aa..1e68d2a61a 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,11 +7,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) export HOST=x86_64-w64-mingw32 -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file" +export DPKG_ADD_ARCH="i386" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false -export RUN_SECURITY_TESTS="true" export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 4644f28a4e..f69afd8a26 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -19,7 +19,7 @@ OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-li OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" + DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 58a8983e6e..82964897e1 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -13,7 +13,7 @@ for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul, echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" - echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b" + echo "( wine \"${b}_orig\" \"\$@\" ) || ( sleep 1 && wine \"${b}_orig\" \"\$@\" )" >> "$b" chmod +x "$b" fi done diff --git a/configure.ac b/configure.ac index 0fa99cfc47..c5bc7ebc7f 100644 --- a/configure.ac +++ b/configure.ac @@ -23,6 +23,7 @@ BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx +BITCOIN_UTIL_NAME=bitcoin-util BITCOIN_WALLET_TOOL_NAME=bitcoin-wallet dnl Multi Process BITCOIN_MP_NODE_NAME=bitcoin-node @@ -131,6 +132,12 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) +AC_ARG_ENABLE([ebpf], + [AS_HELP_STRING([--enable-ebpf], + [enable eBPF tracing (default is yes if sys/sdt.h is found)])], + [use_ebpf=$enableval], + [use_ebpf=yes]) + AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], [enable UPNP (default is yes if libminiupnpc is found)])], @@ -177,10 +184,16 @@ AC_ARG_ENABLE([extended-functional-tests], AC_ARG_ENABLE([fuzz], AS_HELP_STRING([--enable-fuzz], - [enable building of fuzz targets (default no). enabling this will disable all other targets]), + [build for fuzzing (default no). enabling this will disable all other targets and override --{enable,disable}-fuzz-binary]), [enable_fuzz=$enableval], [enable_fuzz=no]) +AC_ARG_ENABLE([fuzz-binary], + AS_HELP_STRING([--enable-fuzz-binary], + [enable building of fuzz binary (default yes).]), + [enable_fuzz_binary=$enableval], + [enable_fuzz_binary=yes]) + AC_ARG_ENABLE([danger_fuzz_link_all], AS_HELP_STRING([--enable-danger-fuzz-link-all], [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]), @@ -325,6 +338,11 @@ AC_ARG_ENABLE([werror], [enable_werror=$enableval], [enable_werror=no]) +AC_ARG_ENABLE([external-signer], + [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is no, requires Boost::Process)])], + [use_external_signer=$enableval], + [use_external_signer=no]) + AC_LANG_PUSH([C++]) dnl Check for a flag to turn compiler warnings into errors. This is helpful for checks which may @@ -420,6 +438,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -571,7 +590,7 @@ CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], - [build bitcoin-cli bitcoin-tx bitcoin-wallet (default=yes)])], + [build bitcoin-cli bitcoin-tx bitcoin-util bitcoin-wallet (default=yes)])], [build_bitcoin_utils=$withval], [build_bitcoin_utils=yes]) @@ -593,6 +612,12 @@ AC_ARG_ENABLE([util-wallet], [build_bitcoin_wallet=$enableval], [build_bitcoin_wallet=$build_bitcoin_utils]) +AC_ARG_ENABLE([util-util], + [AS_HELP_STRING([--enable-util-util], + [build bitcoin-util])], + [build_bitcoin_util=$enableval], + [build_bitcoin_util=$build_bitcoin_utils]) + AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], @@ -637,7 +662,7 @@ case $host in AC_MSG_ERROR("windres not found") fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -793,13 +818,14 @@ if test x$ac_cv_sys_large_files != x && CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -AX_GCC_FUNC_ATTRIBUTE([visibility]) -AX_GCC_FUNC_ATTRIBUTE([dllexport]) -AX_GCC_FUNC_ATTRIBUTE([dllimport]) - if test x$use_glibc_compat != xno; then AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) + case $host in + powerpc64* | ppc64*) + AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) + ;; + esac else AC_SEARCH_LIBS([clock_gettime],[rt]) fi @@ -835,7 +861,10 @@ if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - AX_CHECK_COMPILE_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) + dnl -fcf-protection used with Clang 7 causes ld to emit warnings: + dnl ld: error: ... <corrupt x86 feature size: 0x8> + dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case. + AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) dnl stack-clash-protection does not work properly when building for Windows. dnl We use the test case from https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 @@ -896,6 +925,8 @@ AC_CHECK_DECLS([strnlen]) dnl Check for daemon(3), unrelated to --with-daemon (although used by it) AC_CHECK_DECLS([daemon]) +AC_CHECK_DECLS([pipe2]) + AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include <endian.h> @@ -956,13 +987,13 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [ AC_MSG_RESULT(no)] ) -AC_MSG_CHECKING([for visibility attribute]) -AC_LINK_IFELSE([AC_LANG_SOURCE([ - int foo_def( void ) __attribute__((visibility("default"))); +AC_MSG_CHECKING([for default visibility attribute]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + int foo(void) __attribute__((visibility("default"))); int main(){} ])], [ - AC_DEFINE(HAVE_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) + AC_DEFINE(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) AC_MSG_RESULT(yes) ], [ @@ -973,6 +1004,18 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +AC_MSG_CHECKING([for dllexport attribute]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + __declspec(dllexport) int foo(void); + int main(){} + ])], + [ + AC_DEFINE(HAVE_DLLEXPORT_ATTRIBUTE,1,[Define if the dllexport attribute is supported.]) + AC_MSG_RESULT(yes) + ], + [AC_MSG_RESULT(no)] +) + dnl thread_local is currently disabled when building with glibc back compat. dnl Our minimum supported glibc is 2.17, however support for thread_local dnl did not arrive in glibc until 2.18. @@ -1108,6 +1151,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <fcntl.h>]], [ AC_MSG_RESULT(yes); HAVE_O_CLOEXEC=1 ], [ AC_MSG_RESULT(no); HAVE_O_CLOEXEC=0 ] ) +AC_DEFINE_UNQUOTED([HAVE_O_CLOEXEC], [$HAVE_O_CLOEXEC], [Define to 1 if O_CLOEXEC flag is available.]) dnl crc32c platform checks AC_MSG_CHECKING(for __builtin_prefetch) @@ -1155,12 +1199,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] ) -dnl Check for reduced exports -if test x$use_reduce_exports = xyes; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], - [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) -fi - AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( @@ -1205,10 +1243,11 @@ AC_DEFUN([SUPPRESS_WARNINGS], dnl enable-fuzz should disable all other targets if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets) + AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no + build_bitcoin_util=no build_bitcoin_wallet=no build_bitcoind=no build_bitcoin_libs=no @@ -1217,30 +1256,34 @@ if test "x$enable_fuzz" = "xyes"; then bitcoin_enable_qt_dbus=no enable_wallet=no use_bench=no + use_external_signer=no use_upnp=no use_natpmp=no use_zmq=no + enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AC_MSG_CHECKING([whether main function is needed]) + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], [AC_MSG_RESULT([no])], [AC_MSG_RESULT([yes]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_MAIN_FUNCTION"], + CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], [], [AC_LANG_PROGRAM([[ #include <cstdint> #include <cstddef> extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* unterminated comment to remove the main function ... - ]],[[]])]) + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE([5.5.1]) + BITCOIN_QT_CONFIGURE([5.9.5]) dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES @@ -1249,6 +1292,8 @@ else QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi + + CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" fi if test x$enable_wallet != xno; then @@ -1287,6 +1332,16 @@ if test x$enable_wallet != xno; then fi fi +if test x$use_ebpf != xno; then + AC_CHECK_HEADER([sys/sdt.h], [have_sdt=yes], [have_sdt=no]) +else + have_sdt=no +fi + +if test x$have_sdt = xyes; then + AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints]) +fi + dnl Check for libminiupnpc (optional) if test x$use_upnp != xno; then AC_CHECK_HEADERS( @@ -1324,43 +1379,52 @@ if test "x$use_natpmp" != xno; then fi if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then - use_boost=no + use_boost=no else - use_boost=yes + use_boost=yes fi if test x$use_boost = xyes; then -dnl Minimum required Boost version -define(MINIMUM_REQUIRED_BOOST, 1.58.0) - -dnl Check for Boost libs -AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) -if test x$want_boost = xno; then + dnl Check for Boost headers + AX_BOOST_BASE([1.58.0],[],[AC_MSG_ERROR([Boost is not available!])]) + if test x$want_boost = xno; then AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) -fi -AX_BOOST_SYSTEM -AX_BOOST_FILESYSTEM -AX_BOOST_THREAD - -dnl Opt-in to boost-process -AS_IF([ test x$with_boost_process != x ], [ AX_BOOST_PROCESS ], [ ax_cv_boost_process=no ] ) + fi + AX_BOOST_SYSTEM + AX_BOOST_FILESYSTEM + + dnl Opt-in to Boost Process if external signer support is requested + if test "x$use_external_signer" != xno; then + AC_MSG_CHECKING(for Boost Process) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <boost/process.hpp>]], + [[ boost::process::child* child = new boost::process::child; delete child; ]])], + [ AC_MSG_RESULT(yes) + AC_DEFINE([ENABLE_EXTERNAL_SIGNER],,[define if external signer support is enabled]) + ], + [ AC_MSG_ERROR([Boost::Process is required for external signer support, but not available!])] + ) + fi -if test x$suppress_external_warnings != xno; then + if test x$suppress_external_warnings != xno; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) -fi + fi -dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic -dnl counter implementations. In 1.63 and later the std::atomic approach is default. -m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro -BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" + dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic + dnl counter implementations. In 1.63 and later the std::atomic approach is default. + m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro + BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_THREAD_LIB" + BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB" fi +AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) + +dnl Check for reduced exports if test x$use_reduce_exports = xyes; then - CXXFLAGS="$CXXFLAGS $RE_CXXFLAGS" - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"],, [[$LDFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]]) fi if test x$use_tests = xyes; then @@ -1371,25 +1435,25 @@ if test x$use_tests = xyes; then if test x$use_boost = xyes; then - AX_BOOST_UNIT_TEST_FRAMEWORK - - dnl Determine if -DBOOST_TEST_DYN_LINK is needed - AC_MSG_CHECKING([for dynamic linked boost test]) - TEMP_LIBS="$LIBS" - LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - AC_LINK_IFELSE([AC_LANG_SOURCE([ - #define BOOST_TEST_DYN_LINK - #define BOOST_TEST_MAIN - #include <boost/test/unit_test.hpp> - - ])], - [AC_MSG_RESULT(yes)] - [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], - [AC_MSG_RESULT(no)]) - LIBS="$TEMP_LIBS" - CPPFLAGS="$TEMP_CPPFLAGS" + AX_BOOST_UNIT_TEST_FRAMEWORK + + dnl Determine if -DBOOST_TEST_DYN_LINK is needed + AC_MSG_CHECKING([for dynamic linked boost test]) + TEMP_LIBS="$LIBS" + LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + AC_LINK_IFELSE([AC_LANG_SOURCE([ + #define BOOST_TEST_DYN_LINK + #define BOOST_TEST_MAIN + #include <boost/test/unit_test.hpp> + + ])], + [AC_MSG_RESULT(yes)] + [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], + [AC_MSG_RESULT(no)]) + LIBS="$TEMP_LIBS" + CPPFLAGS="$TEMP_CPPFLAGS" fi fi @@ -1433,7 +1497,7 @@ fi dnl univalue check need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then +if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then need_bundled_univalue=no else if test x$system_univalue != xno; then @@ -1516,6 +1580,10 @@ AC_MSG_CHECKING([whether to build bitcoin-wallet]) AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test x$build_bitcoin_wallet = xyes]) AC_MSG_RESULT($build_bitcoin_wallet) +AC_MSG_CHECKING([whether to build bitcoin-util]) +AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test x$build_bitcoin_util = xyes]) +AC_MSG_RESULT($build_bitcoin_util) + AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) if test x$build_bitcoin_libs = xyes; then @@ -1677,8 +1745,10 @@ AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) +AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) @@ -1695,6 +1765,8 @@ AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) +AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) +AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) @@ -1717,6 +1789,7 @@ AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) +AC_SUBST(BITCOIN_UTIL_NAME) AC_SUBST(BITCOIN_WALLET_TOOL_NAME) AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) @@ -1745,6 +1818,7 @@ AC_SUBST(ARM_CRC_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) AC_SUBST(USE_BDB) +AC_SUBST(ENABLE_EXTERNAL_SIGNER) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) @@ -1820,41 +1894,43 @@ esac echo echo "Options used to compile and link:" -echo " boost process = $ax_cv_boost_process" -echo " multiprocess = $build_multiprocess" -echo " with wallet = $enable_wallet" +echo " external signer = $use_external_signer" +echo " multiprocess = $build_multiprocess" +echo " with libs = $build_bitcoin_libs" +echo " with wallet = $enable_wallet" if test "x$enable_wallet" != "xno"; then - echo " with sqlite = $use_sqlite" - echo " with bdb = $use_bdb" + echo " with sqlite = $use_sqlite" + echo " with bdb = $use_bdb" fi -echo " with gui / qt = $bitcoin_enable_qt" +echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then - echo " with qr = $use_qr" + echo " with qr = $use_qr" fi -echo " with zmq = $use_zmq" +echo " with zmq = $use_zmq" if test x$enable_fuzz == xno; then - echo " with test = $use_tests" + echo " with test = $use_tests" else - echo " with test = not building test_bitcoin because fuzzing is enabled" - echo " with fuzz = $enable_fuzz" + echo " with test = not building test_bitcoin because fuzzing is enabled" + echo " with fuzz = $enable_fuzz" fi -echo " with bench = $use_bench" -echo " with upnp = $use_upnp" -echo " with natpmp = $use_natpmp" -echo " use asm = $use_asm" -echo " sanitizers = $use_sanitizers" -echo " debug enabled = $enable_debug" -echo " gprof enabled = $enable_gprof" -echo " werror = $enable_werror" +echo " with bench = $use_bench" +echo " with upnp = $use_upnp" +echo " with natpmp = $use_natpmp" +echo " use asm = $use_asm" +echo " ebpf tracing = $have_sdt" +echo " sanitizers = $use_sanitizers" +echo " debug enabled = $enable_debug" +echo " gprof enabled = $enable_gprof" +echo " werror = $enable_werror" echo -echo " target os = $TARGET_OS" -echo " build os = $build_os" +echo " target os = $TARGET_OS" +echo " build os = $build_os" echo -echo " CC = $CC" -echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" -echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" -echo " CXX = $CXX" -echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" -echo " ARFLAGS = $ARFLAGS" +echo " CC = $CC" +echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" +echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" +echo " CXX = $CXX" +echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/debian/copyright b/contrib/debian/copyright index a18c5bccc5..7ee7f056d9 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -87,6 +87,10 @@ Files: src/qt/res/icons/proxy.png Copyright: Cristian Mircea Messel License: public-domain +Files: src/qt/fonts/RobotoMono-Bold.ttf +License: Apache-2.0 +Comment: Site: https://fonts.google.com/specimen/Roboto+Mono + License: Expat Permission is hereby granted, free of charge, to any person obtaining a @@ -144,3 +148,14 @@ Comment: License: public-domain This work is in the public domain. + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index b933b264e7..b7bf76ce77 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -14,13 +14,14 @@ BITCOIND=${BITCOIND:-$BINDIR/bitcoind} BITCOINCLI=${BITCOINCLI:-$BINDIR/bitcoin-cli} BITCOINTX=${BITCOINTX:-$BINDIR/bitcoin-tx} WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} +BITCOINUTIL=${BITCOINQT:-$BINDIR/bitcoin-util} BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} [ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 # Don't allow man pages to be generated for binaries built from a dirty tree DIRTY="" -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINQT; do +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do VERSION_OUTPUT=$($cmd --version) if [[ $VERSION_OUTPUT == *"dirty"* ]]; then DIRTY="${DIRTY}${cmd}\n" @@ -43,7 +44,7 @@ read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ prin echo "[COPYRIGHT]" > footer.h2m $BITCOIND --version | sed -n '1!p' >> footer.h2m -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINQT; do +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do cmdname="${cmd##*/}" help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 18ed7d61e0..ee7bfc9805 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -23,29 +23,26 @@ class TestSymbolChecks(unittest.TestCase): executable = 'test1' cc = 'gcc' - # there's no way to do this test for RISC-V at the moment; bionic's libc is 2.27 - # and we allow all symbols from 2.27. - if 'riscv' in get_machine(cc): - self.skipTest("test not available for RISC-V") - - # memfd_create was introduced in GLIBC 2.27, so is newer than the upper limit of - # all but RISC-V but still available on bionic + # renameat2 was introduced in GLIBC 2.28, so is newer than the upper limit + # of glibc for all platforms with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE - #include <sys/mman.h> + #include <stdio.h> + #include <linux/fs.h> - int memfd_create(const char *name, unsigned int flags); + int renameat2(int olddirfd, const char *oldpath, + int newdirfd, const char *newpath, unsigned int flags); int main() { - memfd_create("test", 0); + renameat2(0, "test", 0, "test_", RENAME_EXCHANGE); return 0; } ''') self.assertEqual(call_symbol_check(cc, source, executable, []), - (1, executable + ': symbol memfd_create from unsupported version GLIBC_2.27\n' + + (1, executable + ': symbol renameat2 from unsupported version GLIBC_2.28\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index f105968515..60acb0d593 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -13,15 +13,16 @@ def setup(): programs = ['ruby', 'git', 'make', 'wget', 'curl'] if args.kvm: programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) + elif args.docker: + if not os.path.isfile('/lib/systemd/system/docker.service'): + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install Docker.', file=sys.stderr) + sys.exit(1) else: programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) @@ -34,14 +35,14 @@ def setup(): if not os.path.isdir('bitcoin'): subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] elif not args.kvm: - make_image_prog += ['--lxc'] + make_image_prog += ['--lxc', '--disksize', '13000'] subprocess.check_call(make_image_prog) os.chdir(workdir) - if args.is_bionic and not args.kvm and not args.docker: + if args.is_focal and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') sys.exit(0) @@ -175,7 +176,7 @@ def main(): args = parser.parse_args() workdir = os.getcwd() - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 0e2c5be0fa..b06fc782a3 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-linux-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -15,6 +15,8 @@ packages: - "ca-certificates" - "curl" - "faketime" +- "g++-8" +- "gcc-8" - "git" - "libtool" - "patch" @@ -27,6 +29,12 @@ packages: # - aarch64-linux-gnu - "binutils-aarch64-linux-gnu" - "g++-8-aarch64-linux-gnu" +# - powerpc64-linux-gnu +- "binutils-powerpc64-linux-gnu" +- "g++-8-powerpc64-linux-gnu" +# - powerpc64le-linux-gnu +- "binutils-powerpc64le-linux-gnu" +- "g++-8-powerpc64le-linux-gnu" # - riscv64-linux-gnu - "binutils-riscv64-linux-gnu" - "g++-8-riscv64-linux-gnu" @@ -38,8 +46,8 @@ script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" @@ -64,7 +72,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -78,7 +86,13 @@ script: | echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + if [ "${i:0:11}" = "powerpc64le" ]; then + echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + elif [ "${i:0:9}" = "powerpc64" ]; then + echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + else + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} + fi chmod +x ${WRAP_DIR}/${i}-${prog} fi done @@ -95,7 +109,7 @@ script: | BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 done # Faketime for binaries @@ -118,7 +132,7 @@ script: | # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "riscv64-linux-gnu" ]; then + if [ "${i}" = "powerpc64-linux-gnu" ]; then # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 # TODO: remove this when no longer needed HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" @@ -132,7 +146,7 @@ script: | tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 2330ff7736..3f0c0c3332 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -2,15 +2,19 @@ name: "bitcoin-dmg-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: - "faketime" - "xorriso" +- "python3-pip" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" +- "url": "https://github.com/achow101/signapple.git" + "dir": "signapple" + "commit": "c7e73aa27a7615ac9506559173f787e2906b25eb" files: - "bitcoin-osx-unsigned.tar.gz" script: | @@ -27,15 +31,23 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done - UNSIGNED=bitcoin-osx-unsigned.tar.gz + # Install signapple + cd signapple + python3 -m pip install -U pip setuptools + python3 -m pip install . + export PATH="$HOME/.local/bin":$PATH + cd .. + + UNSIGNED_TARBALL=bitcoin-osx-unsigned.tar.gz + UNSIGNED_APP=dist/Bitcoin-Qt.app SIGNED=bitcoin-osx-signed.dmg - tar -xf ${UNSIGNED} + tar -xf ${UNSIGNED_TARBALL} OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED} signature/osx + ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 9a7dd13c9c..0dc531df0e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-osx-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -29,6 +29,7 @@ packages: - "python3-setuptools" - "fonts-tuffy" - "xorriso" +- "libtinfo5" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -39,7 +40,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg xorrisofs" @@ -63,7 +64,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -75,7 +76,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done @@ -138,8 +139,6 @@ script: | cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate - cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} pushd unsigned-app-${i} find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 6bcd126662..c13c24c3cc 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -2,7 +2,7 @@ name: "bitcoin-win-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index fc79745e69..95cf0185e2 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-win-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -31,7 +31,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g -fno-ident" @@ -55,7 +55,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -67,7 +67,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done @@ -81,7 +81,7 @@ script: | echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 7e28d27454..db28cd07a0 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -1,41 +1,51 @@ 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) -617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa -E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach -152812300785C96444D3334D17565732E08E5E41 Andrew Chow +617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) +E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) +152812300785C96444D3334D17565732E08E5E41 Andrew Chow (achow101) 590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) -912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak +912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) -F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur -C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields -BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random -6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark +18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) +101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) +F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) +C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) +BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) +6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) -D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece -01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen +D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) +6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) +01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) -D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 -82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack -32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli -4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon +A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) +7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) +D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) +82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack (jonatack) +32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) +4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) -E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr -B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke +30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) +E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) 07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) -CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider -E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford +CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider (meshcollider) +E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford (fanquake) AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) -9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo -77E72E69DA7EE0A148C06B21B34821D4944DE5F7 Nils Schneider -F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger -D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy -37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd -D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) -133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo (michagogo) +C57E4B42223FDE851D4F69DD28DF2724F241D8EE midnightmagic (midnightmagic) +F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger (guggero, Oliver Gugger) +D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy (prab) +37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd (petertodd) +D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille [Location: Leuven, Belgium] (sipa) +133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille (sipa) 6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) -ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost +ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) +867345026B6763E8B07EE73AB6737117397F5C4F Stephan Oeste (Emzy) 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) -AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami -79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko -71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan +6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) +AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) +71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index dffcf99607..1122ec9ba5 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -13,11 +13,9 @@ We achieve bootstrappability by using Guix as a functional package manager. Conservatively, a x86_64 machine with: -- 4GB of free disk space on the partition that /gnu/store will reside in -- 24GB of free disk space on the partition that the Bitcoin Core git repository - resides in - -> Note: these requirements are slightly less onerous than those of Gitian builds +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space per platform triple you're planning on building (see + the `HOSTS` environment variable description) ## Setup @@ -40,25 +38,27 @@ Otherwise, follow the [Guix installation guide][guix/bin-install]. Guix allows us to achieve better binary security by using our CPU time to build everything from scratch. However, it doesn't sacrifice user choice in pursuit of -this: users can decide whether or not to bootstrap and to use substitutes. +this: users can decide whether or not to bootstrap and to use substitutes +(pre-built packages). After installation, you may want to consider [adding substitute -servers](#speeding-up-builds-with-substitute-servers) to speed up your build if -that fits your security model (say, if you're just testing that this works). -This is skippable if you're using the [Dockerfile][fanquake/guix-docker]. +servers](#speeding-up-builds-with-substitute-servers) from which to download +pre-built packages to speed up your build if that fits your security model (say, +if you're just testing that this works). Substitute servers are set up by +default if you're using the [Dockerfile][fanquake/guix-docker]. -If you prefer not to use any substitutes, make sure to set -`ADDITIONAL_GUIX_ENVIRONMENT_FLAGS` like the following snippet. The first build -will take a while, but the resulting packages will be cached for future builds. +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. ```sh -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--no-substitutes' +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ``` Likewise, to perform a bootstrapped build (takes even longer): ```sh -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap' ``` ### Using a version of Guix with `guix time-machine` capabilities @@ -82,17 +82,6 @@ export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ## Usage -### As a Development Environment - -For a Bitcoin Core depends development environment, simply invoke - -```sh -guix environment --manifest=contrib/guix/manifest.scm -``` - -And you'll land back in your shell with all the build dependencies required for -a `depends` build injected into your environment. - ### As a Tool for Deterministic Builds From the top of a clean Bitcoin Core repository: @@ -113,10 +102,9 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum * _**HOSTS**_ Override the space-separated list of platform triples for which to perform a - bootstrappable build. _(defaults to "x86\_64-linux-gnu - arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu")_ - - > Windows and OS X platform triplet support are WIP. + bootstrappable build. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf + aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin18")_ * _**SOURCES_PATH**_ @@ -124,6 +112,19 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum depends tree. Setting this to the same directory across multiple builds of the depends tree can eliminate unnecessary redownloading of package sources. +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + +* _**SDK_PATH**_ + + Set the path where _extracted_ SDKs can be found. This is passed through to + the depends tree. Note that this is should be set to the _parent_ directory of + the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of + $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). + * _**MAX_JOBS**_ Override the maximum number of jobs to run simultaneously, you might want to @@ -147,13 +148,29 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum string) is interpreted the same way as not setting `V` at all, and that `V=0` has the same effect as `V=1`. -* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#speeding-up-builds-with-substitute-servers) for more + details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ - Additional flags to be passed to `guix environment`. For a fully-bootstrapped + Additional flags to be passed to all `guix` commands. For a fully-bootstrapped build, set this to `--bootstrap --no-substitutes` (refer to the [security model section](#choosing-your-security-model) for more details). Note that a fully-bootstrapped build will take quite a long time on the first run. +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix environment` inside + `guix time-machine`. + ## Tips and Tricks ### Speeding up builds with substitute servers @@ -161,14 +178,15 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum _This whole section is automatically done in the convenience [Dockerfiles][fanquake/guix-docker]_ -For those who are used to life in the fast _(and trustful)_ lane, you can use -[substitute servers][guix/substitutes] to enable binary downloads of packages. +For those who are used to life in the fast _(and trustful)_ lane, you can +specify [substitute servers][guix/substitutes] from which to download pre-built +packages. > For those who only want to use substitutes from the official Guix build farm > and have authorized the build farm's signing key during Guix's installation, > you don't need to do anything. -#### Authorize the signing keys +#### Step 1: Authorize the signing keys For the official Guix build farm at https://ci.guix.gnu.org, run as root: @@ -182,7 +200,7 @@ For dongcarl's substitute server at https://guix.carldong.io, run as root: wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize ``` -#### Use the substitute servers +#### Step 2: Specify the substitute servers The official Guix build farm at https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag is supplied. @@ -196,7 +214,7 @@ To use dongcarl's substitute server for Bitcoin Core builds after having [authorized his signing key](#authorize-the-signing-keys): ``` -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--substitute-urls="https://guix.carldong.io https://ci.guix.gnu.org"' +export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` ## FAQ @@ -210,11 +228,11 @@ As mentioned at the bottom of [this manual page][guix/bin-install]: > > make guix-binary.x86_64-linux.tar.xz -### When will Guix be packaged in debian? +### Is Guix packaged in my operating system? -Vagrant Cascadian has been making good progress on this -[here][debian/guix-package]. We have all the pieces needed to put up an APT -repository and will likely put one up soon. +Guix is shipped starting with [Debian Bullseye][debian/guix-bullseye] and +[Ubuntu 21.04 "Hirsute Hippo"][ubuntu/guix-hirsute]. Other operating systems +are working on packaging Guix as well. [b17e]: http://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ @@ -226,5 +244,6 @@ repository and will likely put one up soon. [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html -[debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 +[debian/guix-bullseye]: https://packages.debian.org/bullseye/guix +[ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 11d2c8b867..7c32fb5726 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -2,13 +2,158 @@ export LC_ALL=C set -e -o pipefail +################### +## Sanity Checks ## +################### + +################ +# Check 1: Make sure that we can invoke required tools +################ +for cmd in git make guix cat mkdir curl; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi +done + +################ +# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# Check 3: Make sure that we're not in a dirty worktree +################ +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +else + GIT_COMMIT=$(git rev-parse --short=12 HEAD) +fi + +################ +# Check 4: Make sure that build directories do not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86_64-w64-mingw32 + x86_64-apple-darwin18}" + +DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + + mkdir -p "$DISTSRC_BASE" +fi + +################ +# Check 5: When building for darwin, make sure that the macOS SDK exists +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + exit 1 + fi + ;; + esac +done + +######### +# Setup # +######### + # Determine the maximum number of jobs to run simultaneously (overridable by # environment) MAX_JOBS="${MAX_JOBS:-$(nproc)}" +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *linux*) echo linux ;; + *) exit 1 ;; + esac +} + # Download the depends sources now as we won't have internet access in the build # container -make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +for host in $HOSTS; do + make -C "${PWD}/depends" -j"$MAX_JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" @@ -16,11 +161,24 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. time-machine() { + # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ + --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ + --max-jobs="$MAX_JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" } +# Make sure an output directory exists for our builds +OUTDIR="${OUTDIR:-${PWD}/output}" +[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" + +######### +# Build # +######### + # Function to be called when building for host ${1} and the user interrupts the # build int_trap() { @@ -38,9 +196,15 @@ and untracked files and directories will be wiped, allowing you to start anew. EOF } -# Deterministically build Bitcoin Core for HOSTs (overridable by environment) +# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we +# can map them into the container +[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH" +[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE" +[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH" + +# Deterministically build Bitcoin Core # shellcheck disable=SC2153 -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do +for host in $HOSTS; do # Display proper warning when the user interrupts the build trap 'int_trap ${host}' INT @@ -50,6 +214,19 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # for the particular $HOST we're building for export HOST="$host" + # shellcheck disable=SC2030 +cat << EOF +INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${MAX_JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '${OUTDIR:?not set}' + ...bind-mounted in container to: '/outdir' +EOF + # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. # @@ -99,20 +276,47 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # make the downloaded depends sources available to it. The sources # should have been downloaded prior to this invocation. # - # shellcheck disable=SC2086 + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Bitcoin Core) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR"=/outdir \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + ${SDK_PATH:+--share="$SDK_PATH"} \ + --max-jobs="$MAX_JOBS" \ + --keep-failed \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ MAX_JOBS="$MAX_JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR=/outdir \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index d658c4f6a6..b92d3f917d 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -3,6 +3,16 @@ export LC_ALL=C set -e -o pipefail export TZ=UTC +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + if [ -n "$V" ]; then # Print both unexpanded (-v) and expanded (-x) forms of commands as they are # read from this file. @@ -11,9 +21,15 @@ if [ -n "$V" ]; then export VERBOSE="$V" fi -# Check that environment variables assumed to be set by the environment are set -echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." -echo "At most ${MAX_JOBS:?not set} jobs will run at once..." +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + MAX_JOBS: ${MAX_JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF ##################### # Environment Setup # @@ -23,19 +39,6 @@ echo "At most ${MAX_JOBS:?not set} jobs will run at once..." # $HOSTs after successfully building. BASEPREFIX="${PWD}/depends" -# Setup an output directory for our build -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" - -# Setup the directory where our Bitcoin Core build for HOST will occur -DISTSRC="${DISTSRC:-${PWD}/distsrc-${HOST}}" -if [ -e "$DISTSRC" ]; then - echo "DISTSRC directory '${DISTSRC}' exists, probably because of previous builds... Aborting..." - exit 1 -else - mkdir -p "$DISTSRC" -fi - # Given a package name and an output name, return the path of that output in our # current guix environment store_path() { @@ -45,37 +48,64 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Set environment variables to point Guix's cross-toolchain to the right + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export CPATH="${NATIVE_GCC}/include" +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +case "$HOST" in + *darwin*) + # When targeting darwin, some native tools built by depends require + # native packages not incorporated in depends + # + # libcap required by native_cdrkit/wodim + # zlib, bzip2 required by native_cdrkit/genisoimage + for native_pkg in libcap zlib bzip2; do + native_pkg_store_path=$(store_path "$native_pkg") + export LIBRARY_PATH="${native_pkg_store_path}/lib:${LIBRARY_PATH}" + export CPATH="${native_pkg_store_path}/include:${CPATH}" + done + ;; +esac + +# Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in *mingw*) # Determine output paths to use in CROSS_* environment variables CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" - export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" - export CPATH="${NATIVE_GCC}/include" - + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. ;; *linux*) CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" CROSS_GCC="$(store_path "gcc-cross-${HOST}")" - CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because - # the limits.h in it is missing a '#include_next <limits.h>' - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" ;; *) exit 1 ;; @@ -84,14 +114,25 @@ esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" for p in "${PATHS[@]}"; do - if [ ! -d "$p" ]; then + if [ -n "$p" ] && [ ! -d "$p" ]; then echo "'$p' doesn't exist or isn't a directory... Aborting..." exit 1 fi done # Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes +case "$HOST" in + *darwin*) + # The auto-rpath behavior is necessary for darwin builds as some native + # tools built by depends refer to and depend on Guix-built native + # libraries + # + # After the native packages in depends are built, the ld wrapper should + # no longer affect our build, as clang would instead reach for + # x86_64-apple-darwin18-ld from cctools + ;; + *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; +esac # Make /usr/bin if it doesn't exist [ -e /usr/bin ] || mkdir -p /usr/bin @@ -105,12 +146,14 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - *) exit 1 ;; + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + powerpc64-linux-gnu) echo /lib/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib/ld64.so.2;; + *) exit 1 ;; esac ) ;; @@ -121,6 +164,16 @@ export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" export TZ="UTC" +case "$HOST" in + *darwin*) + # cctools AR, unlike GNU binutils AR, does not have a deterministic mode + # or a configure flag to enable determinism by default, it only + # understands if this env-var is set or not. See: + # + # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 + export ZERO_AR_DATE=yes + ;; +esac #################### # Depends Building # @@ -130,6 +183,8 @@ export TZ="UTC" make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ ${V:+V=1} \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ i686_linux_CC=i686-linux-gnu-gcc \ i686_linux_CXX=i686-linux-gnu-g++ \ i686_linux_AR=i686-linux-gnu-ar \ @@ -142,7 +197,8 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + FORCE_USE_SYSTEM_CLANG=1 ########################### @@ -166,7 +222,7 @@ fi ########################### # CONFIGFLAGS -CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" case "$HOST" in *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; esac @@ -176,6 +232,7 @@ HOST_CFLAGS="-O2 -g" case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; esac # CXXFLAGS @@ -187,8 +244,13 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +case "$HOST" in + powerpc64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; +esac + # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +mkdir -p "$DISTSRC" ( cd "$DISTSRC" @@ -205,8 +267,8 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" --disable-maintainer-mode \ --disable-dependency-tracking \ ${CONFIGFLAGS} \ - CFLAGS="${HOST_CFLAGS}" \ - CXXFLAGS="${HOST_CXXFLAGS}" \ + ${HOST_CFLAGS:+CFLAGS="${HOST_CFLAGS}"} \ + ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -216,14 +278,9 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - - case "$HOST" in - *linux*|*mingw*) - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} - ;; - esac + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} # Make the os-specific installers case "$HOST" in @@ -238,8 +295,39 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH - make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + case "$HOST" in + *darwin*) + make install-strip DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + *) + make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + esac + case "$HOST" in + *darwin*) + make osx_volname ${V:+V=1} + make deploydir ${V:+V=1} + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + osx_volname \ + contrib/macdeploy/detached-sig-{apply,create}.sh \ + "${BASEPREFIX}/${HOST}"/native/bin/dmg + for util in codesign_allocate pagestuff; do + cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util" + done + mv --target-directory="unsigned-app-${HOST}" dist + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + ) + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + ;; + esac ( cd installed @@ -254,13 +342,18 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" find . -name "lib*.a" -delete # Prune pkg-config files - rm -r "${DISTNAME}/lib/pkgconfig" + rm -rf "${DISTNAME}/lib/pkgconfig" - # Split binaries and libraries from their debug symbols - { - find "${DISTNAME}/bin" -type f -executable -print0 - find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + case "$HOST" in + *darwin*) ;; + *) + # Split binaries and libraries from their debug symbols + { + find "${DISTNAME}/bin" -type f -executable -print0 + find "${DISTNAME}/lib" -type f -print0 + } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + ;; + esac case "$HOST" in *mingw*) @@ -300,6 +393,13 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) + ;; esac ) ) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 5e011ea184..e0e2e1e16e 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -3,28 +3,44 @@ (gnu packages autotools) (gnu packages base) (gnu packages bash) + (gnu packages cdrom) (gnu packages check) + (gnu packages cmake) (gnu packages commencement) (gnu packages compression) (gnu packages cross-base) (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages gnome) + (gnu packages image) + (gnu packages imagemagick) (gnu packages installers) (gnu packages linux) + (gnu packages llvm) (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) (gnu packages version-control) + (guix build-system font) (guix build-system gnu) (guix build-system trivial) + (guix download) (guix gexp) + ((guix licenses) #:prefix license:) (guix packages) (guix profiles) (guix utils)) +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + (define (make-ssp-fixed-gcc xgcc) "Given a XGCC package, return a modified package that uses the SSP function from glibc instead of from libssp.so. Our `symbol-check' script will complain if @@ -99,7 +115,8 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" `(("binutils" ,xbinutils) ("libc" ,xlibc) ("libc:static" ,xlibc "static") - ("gcc" ,xgcc))) + ("gcc" ,xgcc) + ("gcc-lib" ,xgcc "lib"))) (synopsis (string-append "Complete GCC tool chain for " target)) (description (string-append "This package provides a complete GCC tool chain for " target " development.")) @@ -108,9 +125,9 @@ chain for " target " development.")) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc gcc-5) - (base-kernel-headers linux-libre-headers-4.19) - (base-libc glibc-2.27) + (base-gcc-for-libc gcc-7) + (base-kernel-headers linux-libre-headers-5.4) + (base-libc glibc) ; glibc 2.31 (base-gcc (make-gcc-rpath-link gcc-9))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." @@ -143,18 +160,41 @@ desirable for building Bitcoin Core release binaries." (propagated-inputs `(("binutils" ,xbinutils) ("libc" ,pthreads-xlibc) - ("gcc" ,pthreads-xgcc))) + ("gcc" ,pthreads-xgcc) + ("gcc-lib" ,pthreads-xgcc "lib"))) (synopsis (string-append "Complete GCC tool chain for " target)) (description (string-append "This package provides a complete GCC tool chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) +(define (make-nsis-with-sde-support base-nsis) + (package-with-extra-patches base-nsis + (search-our-patches "nsis-SConstruct-sde-support.patch"))) + +(define-public font-tuffy + (package + (name "font-tuffy") + (version "20120614") + (source + (origin + (method url-fetch) + (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) + (build-system font-build-system) + (home-page "http://tulrich.com/fonts/") + (synopsis "The Tuffy Truetype Font Family") + (description + "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") + (license license:public-domain))) (packages->manifest (append (list ;; The Basics - bash-minimal + bash which coreutils util-linux @@ -181,18 +221,19 @@ chain for " target " development.")) pkg-config ;; Scripting perl - python-3.7 + python-3 ;; Git git - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27)) + ;; Native gcc 7 toolchain + gcc-toolchain-7) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows - (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) - ((string-contains target "riscv64-linux-") - (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" - #:base-gcc-for-libc gcc-7))) + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (make-nsis-with-sde-support nsis-x86_64))) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) + ((string-contains target "darwin") + (list clang-8 libcap binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) (else '()))))) diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch new file mode 100644 index 0000000000..5edf1b7c8e --- /dev/null +++ b/contrib/guix/patches/nsis-SConstruct-sde-support.patch @@ -0,0 +1,15 @@ +diff --git a/SConstruct b/SConstruct +index e8252c9..41786f2 100755 +--- a/SConstruct ++++ b/SConstruct +@@ -95,8 +95,8 @@ default_doctype = 'html' + if defenv.WhereIs('hhc', os.environ['PATH']): + default_doctype = 'chm' + +-from time import strftime, gmtime +-cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) ++import time ++cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) + + opts = Variables() + diff --git a/contrib/init/README.md b/contrib/init/README.md index 306a37f75a..affc7c2e75 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -1,6 +1,6 @@ Sample configuration files for: ``` -SystemD: bitcoind.service +systemd: bitcoind.service Upstart: bitcoind.conf OpenRC: bitcoind.openrc bitcoind.openrcconf @@ -9,4 +9,4 @@ macOS: org.bitcoin.bitcoind.plist ``` have been made available to assist packagers in creating node packages here. -See doc/init.md for more information. +See [doc/init.md](../../doc/init.md) for more information. diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 8b308644b1..5999928aa4 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -11,7 +11,11 @@ [Unit] Description=Bitcoin daemon -After=network.target +Documentation=https://github.com/bitcoin/bitcoin/blob/master/doc/init.md + +# https://www.freedesktop.org/wiki/Software/systemd/NetworkTarget/ +After=network-online.target +Wants=network-online.target [Service] ExecStart=/usr/bin/bitcoind -daemon \ diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index e9130a21de..4037936404 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -68,10 +68,155 @@ tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" cd "${BDB_PREFIX}/${BDB_VERSION}/" # Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) -CLANG_CXX11_PATCH_URL='https://gist.githubusercontent.com/LnL7/5153b251fd525fe15de69b67e63a6075/raw/7778e9364679093a32dec2908656738e16b6bdcb/clang.patch' -CLANG_CXX11_PATCH_HASH='7a9a47b03fd5fb93a16ef42235fa9512db9b0829cfc3bdf90edd3ec1f44d637c' -http_get "${CLANG_CXX11_PATCH_URL}" clang.patch "${CLANG_CXX11_PATCH_HASH}" -patch -p2 < clang.patch +patch --ignore-whitespace -p1 << 'EOF' +commit 3311d68f11d1697565401eee6efc85c34f022ea7 +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:03:56 2020 +0800 + + Fix C++11 compatibility + +diff --git a/dbinc/atomic.h b/dbinc/atomic.h +index 0034dcc..7c11d4a 100644 +--- a/dbinc/atomic.h ++++ b/dbinc/atomic.h +@@ -70,7 +70,7 @@ typedef struct { + * These have no memory barriers; the caller must include them when necessary. + */ + #define atomic_read(p) ((p)->value) +-#define atomic_init(p, val) ((p)->value = (val)) ++#define atomic_init_db(p, val) ((p)->value = (val)) + + #ifdef HAVE_ATOMIC_SUPPORT + +@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; + #define atomic_inc(env, p) __atomic_inc(p) + #define atomic_dec(env, p) __atomic_dec(p) + #define atomic_compare_exchange(env, p, o, n) \ +- __atomic_compare_exchange((p), (o), (n)) ++ __atomic_compare_exchange_db((p), (o), (n)) + static inline int __atomic_inc(db_atomic_t *p) + { + int temp; +@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) + * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html + * which configure could be changed to use. + */ +-static inline int __atomic_compare_exchange( ++static inline int __atomic_compare_exchange_db( + db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) + { + atomic_value_t was; +@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( + #define atomic_dec(env, p) (--(p)->value) + #define atomic_compare_exchange(env, p, oldval, newval) \ + (DB_ASSERT(env, atomic_read(p) == (oldval)), \ +- atomic_init(p, (newval)), 1) ++ atomic_init_db(p, (newval)), 1) + #else + #define atomic_inc(env, p) __atomic_inc(env, p) + #define atomic_dec(env, p) __atomic_dec(env, p) +diff --git a/mp/mp_fget.c b/mp/mp_fget.c +index 5fdee5a..0b75f57 100644 +--- a/mp/mp_fget.c ++++ b/mp/mp_fget.c +@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ + + /* Initialize enough so we can call __memp_bhfree. */ + alloc_bhp->flags = 0; +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + #ifdef DIAGNOSTIC + if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { + __db_errx(env, +@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ + MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, + PROT_READ); + +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + MUTEX_LOCK(env, alloc_bhp->mtx_buf); + alloc_bhp->priority = bhp->priority; + alloc_bhp->pgno = bhp->pgno; +diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c +index 34467d2..f05aa0c 100644 +--- a/mp/mp_mvcc.c ++++ b/mp/mp_mvcc.c +@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) + #else + memcpy(frozen_bhp, bhp, SSZA(BH, buf)); + #endif +- atomic_init(&frozen_bhp->ref, 0); ++ atomic_init_db(&frozen_bhp->ref, 0); + if (mutex != MUTEX_INVALID) + frozen_bhp->mtx_buf = mutex; + else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, +@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) + #endif + alloc_bhp->mtx_buf = mutex; + MUTEX_LOCK(env, alloc_bhp->mtx_buf); +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + F_CLR(alloc_bhp, BH_FROZEN); + } + +diff --git a/mp/mp_region.c b/mp/mp_region.c +index e6cece9..ddbe906 100644 +--- a/mp/mp_region.c ++++ b/mp/mp_region.c +@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) + return (ret); + SH_TAILQ_INIT(&htab[i].hash_bucket); +- atomic_init(&htab[i].hash_page_dirty, 0); ++ atomic_init_db(&htab[i].hash_page_dirty, 0); + } + + /* +@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : + mtx_base + i; + SH_TAILQ_INIT(&hp->hash_bucket); +- atomic_init(&hp->hash_page_dirty, 0); ++ atomic_init_db(&hp->hash_page_dirty, 0); + #ifdef HAVE_STATISTICS + hp->hash_io_wait = 0; + hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; +diff --git a/mutex/mut_method.c b/mutex/mut_method.c +index 2588763..5c6d516 100644 +--- a/mutex/mut_method.c ++++ b/mutex/mut_method.c +@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) + MUTEX_LOCK(env, mtx); + ret = atomic_read(v) == oldval; + if (ret) +- atomic_init(v, newval); ++ atomic_init_db(v, newval); + MUTEX_UNLOCK(env, mtx); + + return (ret); +diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c +index f3922e0..e40fcdf 100644 +--- a/mutex/mut_tas.c ++++ b/mutex/mut_tas.c +@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) + + #ifdef HAVE_SHARED_LATCHES + if (F_ISSET(mutexp, DB_MUTEX_SHARED)) +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + else + #endif + if (MUTEX_INIT(&mutexp->tas)) { +@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) + F_CLR(mutexp, DB_MUTEX_LOCKED); + /* Flush flag update before zeroing count */ + MEMBAR_EXIT(); +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + } else { + DB_ASSERT(env, sharecount > 0); + MEMBAR_EXIT(); +EOF # The packaged config.guess and config.sub are ancient (2009) and can cause build issues. # Replace them with modern versions. diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index 5c5a85d3fe..d481413cc3 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -8,10 +8,9 @@ set -e UNSIGNED="$1" SIGNATURE="$2" -ARCH=x86_64 ROOTDIR=dist -TEMPDIR=signed.temp OUTDIR=signed-app +SIGNAPPLE=signapple if [ -z "$UNSIGNED" ]; then echo "usage: $0 <unsigned app> <signature>" @@ -23,35 +22,6 @@ if [ -z "$SIGNATURE" ]; then exit 1 fi -rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR} -tar -C ${TEMPDIR} -xf ${UNSIGNED} -cp -rf "${SIGNATURE}"/* ${TEMPDIR} - -if [ -z "${PAGESTUFF}" ]; then - PAGESTUFF=${TEMPDIR}/pagestuff -fi - -if [ -z "${CODESIGN_ALLOCATE}" ]; then - CODESIGN_ALLOCATE=${TEMPDIR}/codesign_allocate -fi - -find ${TEMPDIR} -name "*.sign" | while read i; do - SIZE=$(stat -c %s "${i}") - TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')" - - echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}" - ${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp" - - OFFSET=$(${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') - if [ -z ${QUIET} ]; then - echo "Attaching signature at offset ${OFFSET}" - fi - - dd if="$i" of="${i}.tmp" bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null - mv "${i}.tmp" "${TARGET_FILE}" - rm "${i}" - echo "Success." -done -mv ${TEMPDIR}/${ROOTDIR} ${OUTDIR} -rm -rf ${TEMPDIR} +${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} +mv ${ROOTDIR} ${OUTDIR} echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 31a97f0a24..4f246cbb3f 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -8,44 +8,21 @@ set -e ROOTDIR=dist BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" -CODESIGN=codesign +SIGNAPPLE=signapple TEMPDIR=sign.temp -TEMPLIST=${TEMPDIR}/signatures.txt OUT=signature-osx.tar.gz -OUTROOT=osx +OUTROOT=osx/dist if [ -z "$1" ]; then - echo "usage: $0 <codesign args>" - echo "example: $0 -s MyIdentity" + echo "usage: $0 <signapple args>" + echo "example: $0 <path to key>" exit 1 fi -rm -rf ${TEMPDIR} ${TEMPLIST} +rm -rf ${TEMPDIR} mkdir -p ${TEMPDIR} -${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}" - -grep -v CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" - SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g') - OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') - SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign" - DIRNAME="$(dirname "${SIGNFILE}")" - mkdir -p "${DIRNAME}" - echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}" - dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null -done - -grep CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" - RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" - DIRNAME="$(dirname "${RESOURCE}")" - mkdir -p "${DIRNAME}" - echo "Adding resource for: \"${TARGETFILE}\"" - cp "${i}" "${RESOURCE}" -done - -rm ${TEMPLIST} +${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" tar -C "${TEMPDIR}" -czf "${OUT}" . rm -rf "${TEMPDIR}" diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md new file mode 100644 index 0000000000..7301968461 --- /dev/null +++ b/contrib/message-capture/message-capture-docs.md @@ -0,0 +1,25 @@ +# Per-Peer Message Capture + +## Purpose + +This feature allows for message capture on a per-peer basis. It answers the simple question: "Can I see what messages my node is sending and receiving?" + +## Usage and Functionality + +* Run `bitcoind` with the `-capturemessages` option. +* Look in the `message_capture` folder in your datadir. + * Typically this will be `~/.bitcoin/message_capture`. + * See that there are many folders inside, one for each peer names with its IP address and port. + * Inside each peer's folder there are two `.dat` files: one is for received messages (`msgs_recv.dat`) and the other is for sent messages (`msgs_sent.dat`). +* Run `contrib/message-capture/message-capture-parser.py` with the proper arguments. + * See the `-h` option for help. + * To see all messages, both sent and received, for all peers use: + ``` + ./contrib/message-capture/message-capture-parser.py -o out.json \ + ~/.bitcoin/message_capture/**/*.dat + ``` + * Note: The messages in the given `.dat` files will be interleaved in chronological order. So, giving both received and sent `.dat` files (as above with `*.dat`) will result in all messages being interleaved in chronological order. + * If an output file is not provided (i.e. the `-o` option is not used), then the output prints to `stdout`. +* View the resulting output. + * The output file is `JSON` formatted. + * Suggestion: use `jq` to view the output, with `jq . out.json` diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py new file mode 100755 index 0000000000..9988478f1b --- /dev/null +++ b/contrib/message-capture/message-capture-parser.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Parse message capture binary files. To be used in conjunction with -capturemessages.""" + +import argparse +import os +import shutil +import sys +from io import BytesIO +import json +from pathlib import Path +from typing import Any, List, Optional + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.messages import ser_uint256 # noqa: E402 +from test_framework.p2p import MESSAGEMAP # noqa: E402 + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +# The test framework classes stores hashes as large ints in many cases. +# These are variables of type uint256 in core. +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. +# As such, they are itemized here. +# Any variables with these names that are of type int are actually uint256 variables. +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) +HASH_INTS = [ + "blockhash", + "block_hash", + "hash", + "hashMerkleRoot", + "hashPrevBlock", + "hashstop", + "prev_header", + "sha256", + "stop_hash", +] + +HASH_INT_VECTORS = [ + "hashes", + "headers", + "vHave", + "vHash", +] + + +class ProgressBar: + def __init__(self, total: float): + self.total = total + self.running = 0 + + def set_progress(self, progress: float): + cols = shutil.get_terminal_size()[0] + if cols <= 12: + return + max_blocks = cols - 9 + num_blocks = int(max_blocks * progress) + print('\r[ {}{} ] {:3.0f}%' + .format('#' * num_blocks, + ' ' * (max_blocks - num_blocks), + progress * 100), + end ='') + + def update(self, more: float): + self.running += more + self.set_progress(self.running / self.total) + + +def to_jsonable(obj: Any) -> Any: + if hasattr(obj, "__dict__"): + return obj.__dict__ + elif hasattr(obj, "__slots__"): + ret = {} # type: Any + for slot in obj.__slots__: + val = getattr(obj, slot, None) + if slot in HASH_INTS and isinstance(val, int): + ret[slot] = ser_uint256(val).hex() + elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + ret[slot] = [ser_uint256(a).hex() for a in val] + else: + ret[slot] = to_jsonable(val) + return ret + elif isinstance(obj, list): + return [to_jsonable(a) for a in obj] + elif isinstance(obj, bytes): + return obj.hex() + else: + return obj + + +def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: + with open(path, 'rb') as f_in: + if progress_bar: + bytes_read = 0 + + while True: + if progress_bar: + # Update progress bar + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + bytes_read = f_in.tell() - 1 + + # Read the Header + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + msgtype = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] # type: bytes + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + + # Start converting the message to a dictionary + msg_dict = {} + msg_dict["direction"] = "recv" if recv else "sent" + msg_dict["time"] = time + msg_dict["size"] = length # "size" is less readable here, but more readable in the output + + msg_ser = BytesIO(f_in.read(length)) + + # Determine message type + if msgtype not in MESSAGEMAP: + # Unrecognized message type + try: + msgtype_tmp = msgtype.decode() + if not msgtype_tmp.isprintable(): + raise UnicodeDecodeError + msg_dict["msgtype"] = msgtype_tmp + except UnicodeDecodeError: + msg_dict["msgtype"] = "UNREADABLE" + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unrecognized message type." + messages.append(msg_dict) + print(f"WARNING - Unrecognized message type {msgtype} in {path}", file=sys.stderr) + continue + + # Deserialize the message + msg = MESSAGEMAP[msgtype]() + msg_dict["msgtype"] = msgtype.decode() + + try: + msg.deserialize(msg_ser) + except KeyboardInterrupt: + raise + except Exception: + # Unable to deserialize message body + msg_ser.seek(0, os.SEEK_SET) + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unable to deserialize message." + messages.append(msg_dict) + print(f"WARNING - Unable to deserialize message in {path}", file=sys.stderr) + continue + + # Convert body of message into a jsonable object + if length: + msg_dict["body"] = to_jsonable(msg) + messages.append(msg_dict) + + if progress_bar: + # Update the progress bar to the end of the current file + # in case we exited the loop early + f_in.seek(0, os.SEEK_END) # Go to end of file + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format(sys.argv[0]), + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "capturepaths", + nargs='+', + help="binary message capture files to parse.") + parser.add_argument( + "-o", "--output", + help="output file. If unset print to stdout") + parser.add_argument( + "-n", "--no-progress-bar", + action='store_true', + help="disable the progress bar. Automatically set if the output is not a terminal") + args = parser.parse_args() + capturepaths = [Path.cwd() / Path(capturepath) for capturepath in args.capturepaths] + output = Path.cwd() / Path(args.output) if args.output else False + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() + + messages = [] # type: List[Any] + if use_progress_bar: + total_size = sum(capture.stat().st_size for capture in capturepaths) + progress_bar = ProgressBar(total_size) + else: + progress_bar = None + + for capture in capturepaths: + process_file(str(capture), messages, "recv" in capture.stem, progress_bar) + + messages.sort(key=lambda msg: msg['time']) + + if use_progress_bar: + progress_bar.set_progress(1) + + jsonrep = json.dumps(messages) + if output: + with open(str(output), 'w+', encoding="utf8") as f_out: + f_out.write(jsonrep) + else: + print(jsonrep) + +if __name__ == "__main__": + main() diff --git a/contrib/signet/README.md b/contrib/signet/README.md index c4aa5ae2f7..71dc2f9638 100644 --- a/contrib/signet/README.md +++ b/contrib/signet/README.md @@ -17,3 +17,64 @@ Syntax: `getcoins.py [-h|--help] [-c|--cmd=<bitcoin-cli path>] [-f|--faucet=<fau If using the default network, invoking the script with no arguments should be sufficient under normal circumstances, but if multiple people are behind the same IP address, the faucet will by default only accept one claim per day. See `--password` above. + +miner +===== + +To mine the first block in your custom chain, you can run: + + cd src/ + CLI="./bitcoin-cli -conf=mysignet.conf" + MINER="..contrib/signet/miner" + GRIND="./bitcoin-util grind" + ADDR=$($CLI -signet getnewaddress) + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --set-block-time=-1 + +This will mine a block with the current timestamp. If you want to backdate the chain, you can give a different timestamp to --set-block-time. + +You will then need to pick a difficulty target. Since signet chains are primarily protected by a signature rather than proof of work, there is no need to spend as much energy as possible mining, however you may wish to choose to spend more time than the absolute minimum. The calibrate subcommand can be used to pick a target, eg: + + $MINER calibrate --grind-cmd="$GRIND" + nbits=1e00f403 for 25s average mining time + +It defaults to estimating an nbits value resulting in 25s average time to find a block, but the --seconds parameter can be used to pick a different target, or the --nbits parameter can be used to estimate how long it will take for a given difficulty. + +Using the --ongoing parameter will then cause the signet miner to create blocks indefinitely. It will pick the time between blocks so that difficulty is adjusted to match the provided --nbits value. + + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=1e00f403 --ongoing + +Other options +------------- + +The --debug and --quiet options are available to control how noisy the signet miner's output is. Note that the --debug, --quiet and --cli parameters must all appear before the subcommand (generate, calibrate, etc) if used. + +Instead of specifying --ongoing, you can specify --max-blocks=N to mine N blocks and stop. + +Instead of using a single address, a ranged descriptor may be provided instead (via the --descriptor parameter), with the reward for the block at height H being sent to the H'th address generated from the descriptor. + +Instead of calculating a specific nbits value, --min-nbits can be specified instead, in which case the mininmum signet difficulty will be targeted. + +By default, the signet miner mines blocks at fixed intervals with minimal variation. If you want blocks to appear more randomly, as they do in mainnet, specify the --poisson option. + +Using the --multiminer parameter allows mining to be distributed amongst multiple miners. For example, if you have 3 miners and want to share blocks between them, specify --multiminer=1/3 on one, --multiminer=2/3 on another, and --multiminer=3/3 on the last one. If you want one to do 10% of blocks and two others to do 45% each, --multiminer=1-10/100 on the first, and --multiminer=11-55 and --multiminer=56-100 on the others. Note that which miner mines which block is determined by the previous block hash, so occasional runs of one miner doing many blocks in a row is to be expected. + +When --multiminer is used, if a miner is down and does not mine a block within five minutes of when it is due, the other miners will automatically act as redundant backups ensuring the chain does not halt. The --backup-delay parameter can be used to change how long a given miner waits, allowing one to be the primary backup (after five minutes) and another to be the secondary backup (after six minutes, eg). + +The --standby-delay parameter can be used to make a backup miner that only mines if a block doesn't arrive on time. This can be combined with --multiminer if desired. Setting --standby-delay also prevents the first block from being mined immediately. + +Advanced usage +-------------- + +The process generate follows internally is to get a block template, convert that into a PSBT, sign the PSBT, move the signature from the signed PSBT into the block template's coinbase, grind proof of work for the block, and then submit the block to the network. + +These steps can instead be done explicitly: + + $CLI -signet getblocktemplate '{"rules": ["signet","segwit"]}' | + $MINER --cli="$CLI" genpsbt --address="$ADDR" | + $CLI -signet -stdin walletprocesspsbt | + jq -r .psbt | + $MINER --cli="$CLI" solvepsbt --grind-cmd="$GRIND" | + $CLI -signet -stdin submitblock + +This is intended to allow you to replace part of the pipeline for further experimentation, if desired. + diff --git a/contrib/signet/miner b/contrib/signet/miner new file mode 100755 index 0000000000..a3fba49d0e --- /dev/null +++ b/contrib/signet/miner @@ -0,0 +1,639 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import base64 +import json +import logging +import math +import os.path +import re +import struct +import sys +import time +import subprocess + +from binascii import unhexlify +from io import BytesIO + +PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) +sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) + +from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, ToHex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, uint256_from_str # noqa: E402 +from test_framework.script import CScriptOp # noqa: E402 + +logging.basicConfig( + format='%(asctime)s %(levelname)s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S') + +SIGNET_HEADER = b"\xec\xc7\xda\xa2" +PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed +RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") + +# #### some helpers that could go into test_framework + +# like FromHex, but without the hex part +def FromBinary(cls, stream): + """deserialize a binary stream (or bytes object) into an object""" + # handle bytes object by turning it into a stream + was_bytes = isinstance(stream, bytes) + if was_bytes: + stream = BytesIO(stream) + obj = cls() + obj.deserialize(stream) + if was_bytes: + assert len(stream.read()) == 0 + return obj + +class PSBTMap: + """Class for serializing and deserializing PSBT maps""" + + def __init__(self, map=None): + self.map = map if map is not None else {} + + def deserialize(self, f): + m = {} + while True: + k = deser_string(f) + if len(k) == 0: + break + v = deser_string(f) + if len(k) == 1: + k = k[0] + assert k not in m + m[k] = v + self.map = m + + def serialize(self): + m = b"" + for k,v in self.map.items(): + if isinstance(k, int) and 0 <= k and k <= 255: + k = bytes([k]) + m += ser_compact_size(len(k)) + k + m += ser_compact_size(len(v)) + v + m += b"\x00" + return m + +class PSBT: + """Class for serializing and deserializing PSBTs""" + + def __init__(self): + self.g = PSBTMap() + self.i = [] + self.o = [] + self.tx = None + + def deserialize(self, f): + assert f.read(5) == b"psbt\xff" + self.g = FromBinary(PSBTMap, f) + assert 0 in self.g.map + self.tx = FromBinary(CTransaction, self.g.map[0]) + self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] + self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] + return self + + def serialize(self): + assert isinstance(self.g, PSBTMap) + assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) + assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) + assert 0 in self.g.map + tx = FromBinary(CTransaction, self.g.map[0]) + assert len(tx.vin) == len(self.i) + assert len(tx.vout) == len(self.o) + + psbt = [x.serialize() for x in [self.g] + self.i + self.o] + return b"psbt\xff" + b"".join(psbt) + + def to_base64(self): + return base64.b64encode(self.serialize()).decode("utf8") + + @classmethod + def from_base64(cls, b64psbt): + return FromBinary(cls, base64.b64decode(b64psbt)) + +# ##### + +def create_coinbase(height, value, spk): + cb = CTransaction() + cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] + cb.vout = [CTxOut(value, spk)] + return cb + +def get_witness_script(witness_root, witness_nonce): + commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) + return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) + +def signet_txs(block, challenge): + # assumes signet solution has not been added yet so does not need + # to be removed + + txs = block.vtx[:] + txs[0] = CTransaction(txs[0]) + txs[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER) + hashes = [] + for tx in txs: + tx.rehash() + hashes.append(ser_uint256(tx.sha256)) + mroot = block.get_merkle_root(hashes) + + sd = b"" + sd += struct.pack("<i", block.nVersion) + sd += ser_uint256(block.hashPrevBlock) + sd += ser_uint256(mroot) + sd += struct.pack("<I", block.nTime) + + to_spend = CTransaction() + to_spend.nVersion = 0 + to_spend.nLockTime = 0 + to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] + to_spend.vout = [CTxOut(0, challenge)] + to_spend.rehash() + + spend = CTransaction() + spend.nVersion = 0 + spend.nLockTime = 0 + spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] + spend.vout = [CTxOut(0, b"\x6a")] + + return spend, to_spend + +def do_createpsbt(block, signme, spendme): + psbt = PSBT() + psbt.g = PSBTMap( {0: signme.serialize(), + PSBT_SIGNET_BLOCK: block.serialize() + } ) + psbt.i = [ PSBTMap( {0: spendme.serialize(), + 3: bytes([1,0,0,0])}) + ] + psbt.o = [ PSBTMap() ] + return psbt.to_base64() + +def do_decode_psbt(b64psbt): + psbt = PSBT.from_base64(b64psbt) + + assert len(psbt.tx.vin) == 1 + assert len(psbt.tx.vout) == 1 + assert PSBT_SIGNET_BLOCK in psbt.g.map + + scriptSig = psbt.i[0].map.get(7, b"") + scriptWitness = psbt.i[0].map.get(8, b"\x00") + + return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + +def finish_block(block, signet_solution, grind_cmd): + block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) + block.vtx[0].rehash() + block.hashMerkleRoot = block.calc_merkle_root() + if grind_cmd is None: + block.solve() + else: + headhex = CBlockHeader.serialize(block).hex() + cmd = grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + newhead = FromHex(CBlockHeader(), newheadhex.decode('utf8')) + block.nNonce = newhead.nNonce + block.rehash() + return block + +def generate_psbt(tmpl, reward_spk, *, blocktime=None): + signet_spk = tmpl["signet_challenge"] + signet_spk_bin = unhexlify(signet_spk) + + cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) + cbtx.vin[0].nSequence = 2**32-2 + cbtx.rehash() + + block = CBlock() + block.nVersion = tmpl["version"] + block.hashPrevBlock = int(tmpl["previousblockhash"], 16) + block.nTime = tmpl["curtime"] if blocktime is None else blocktime + if block.nTime < tmpl["mintime"]: + block.nTime = tmpl["mintime"] + block.nBits = int(tmpl["bits"], 16) + block.nNonce = 0 + block.vtx = [cbtx] + [FromHex(CTransaction(), t["data"]) for t in tmpl["transactions"]] + + witnonce = 0 + witroot = block.calc_witness_merkle_root() + cbwit = CTxInWitness() + cbwit.scriptWitness.stack = [ser_uint256(witnonce)] + block.vtx[0].wit.vtxinwit = [cbwit] + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + + signme, spendme = signet_txs(block, signet_spk_bin) + + return do_createpsbt(block, signme, spendme) + +def get_reward_address(args, height): + if args.address is not None: + return args.address + + if '*' not in args.descriptor: + addr = json.loads(args.bcli("deriveaddresses", args.descriptor))[0] + args.address = addr + return addr + + remove = [k for k in args.derived_addresses.keys() if k+20 <= height] + for k in remove: + del args.derived_addresses[k] + + addr = args.derived_addresses.get(height, None) + if addr is None: + addrs = json.loads(args.bcli("deriveaddresses", args.descriptor, "[%d,%d]" % (height, height+20))) + addr = addrs[0] + for k, a in enumerate(addrs): + args.derived_addresses[height+k] = a + + return addr + +def get_reward_addr_spk(args, height): + assert args.address is not None or args.descriptor is not None + + if hasattr(args, "reward_spk"): + return args.address, args.reward_spk + + reward_addr = get_reward_address(args, height) + reward_spk = unhexlify(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + if args.address is not None: + # will always be the same, so cache + args.reward_spk = reward_spk + + return reward_addr, reward_spk + +def do_genpsbt(args): + tmpl = json.load(sys.stdin) + _, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + psbt = generate_psbt(tmpl, reward_spk) + print(psbt) + +def do_solvepsbt(args): + block, signet_solution = do_decode_psbt(sys.stdin.read()) + block = finish_block(block, signet_solution, args.grind_cmd) + print(ToHex(block)) + +def nbits_to_target(nbits): + shift = (nbits >> 24) & 0xff + return (nbits & 0x00ffffff) * 2**(8*(shift - 3)) + +def target_to_nbits(target): + tstr = "{0:x}".format(target) + if len(tstr) < 6: + tstr = ("000000"+tstr)[-6:] + if len(tstr) % 2 != 0: + tstr = "0" + tstr + if int(tstr[0],16) >= 0x8: + # avoid "negative" + tstr = "00" + tstr + fix = int(tstr[:6], 16) + sz = len(tstr)//2 + if tstr[6:] != "0"*(sz*2-6): + fix += 1 + + return int("%02x%06x" % (sz,fix), 16) + +def seconds_to_hms(s): + if s == 0: + return "0s" + neg = (s < 0) + if neg: + s = -s + out = "" + if s % 60 > 0: + out = "%ds" % (s % 60) + s //= 60 + if s % 60 > 0: + out = "%dm%s" % (s % 60, out) + s //= 60 + if s > 0: + out = "%dh%s" % (s, out) + if neg: + out = "-" + out + return out + +def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): + # strategy: + # 1) work out how far off our desired target we are + # 2) cap it to a factor of 4 since that's the best we can do in a single retarget period + # 3) use that to work out the desired average interval in this retarget period + # 4) if doing poisson, use the last hash to pick a uniformly random number in [0,1), and work out a random multiplier to vary the average by + # 5) cap the resulting interval between 1 second and 1 hour to avoid extremes + + INTERVAL = 600.0*2016/2015 # 10 minutes, adjusted for the off-by-one bug + + current_target = nbits_to_target(last_nbits) + retarget_factor = ultimate_target / current_target + retarget_factor = max(0.25, min(retarget_factor, 4.0)) + + avg_interval = INTERVAL * retarget_factor + + if do_poisson: + det_rand = int(last_hash[-8:], 16) * 2**-32 + this_interval_variance = -math.log1p(-det_rand) + else: + this_interval_variance = 1 + + this_interval = avg_interval * this_interval_variance + this_interval = max(1, min(this_interval, 3600)) + + return this_interval + +def next_block_is_mine(last_hash, my_blocks): + det_rand = int(last_hash[-16:-8], 16) + return my_blocks[0] <= (det_rand % my_blocks[2]) < my_blocks[1] + +def do_generate(args): + if args.max_blocks is not None: + if args.ongoing: + logging.error("Cannot specify both --ongoing and --max-blocks") + return 1 + if args.max_blocks < 1: + logging.error("N must be a positive integer") + return 1 + max_blocks = args.max_blocks + elif args.ongoing: + max_blocks = None + else: + max_blocks = 1 + + if args.set_block_time is not None and max_blocks != 1: + logging.error("Cannot specify --ongoing or --max-blocks > 1 when using --set-block-time") + return 1 + if args.set_block_time is not None and args.set_block_time < 0: + args.set_block_time = time.time() + logging.info("Treating negative block time as current time (%d)" % (args.set_block_time)) + + if args.min_nbits: + if args.nbits is not None: + logging.error("Cannot specify --nbits and --min-nbits") + return 1 + args.nbits = "1e0377ae" + logging.info("Using nbits=%s" % (args.nbits)) + + if args.set_block_time is None: + if args.nbits is None or len(args.nbits) != 8: + logging.error("Must specify --nbits (use calibrate command to determine value)") + return 1 + + if args.multiminer is None: + my_blocks = (0,1,1) + else: + if not args.ongoing: + logging.error("Cannot specify --multiminer without --ongoing") + return 1 + m = RE_MULTIMINER.match(args.multiminer) + if m is None: + logging.error("--multiminer argument must be k/m or j-k/m") + return 1 + start,_,stop,total = m.groups() + if stop is None: + stop = start + start, stop, total = map(int, (start, stop, total)) + if stop < start or start <= 0 or total < stop or total == 0: + logging.error("Inconsistent values for --multiminer") + return 1 + my_blocks = (start-1, stop, total) + + ultimate_target = nbits_to_target(int(args.nbits,16)) + + mined_blocks = 0 + bestheader = {"hash": None} + lastheader = None + while max_blocks is None or mined_blocks < max_blocks: + + # current status? + bci = json.loads(args.bcli("getblockchaininfo")) + + if bestheader["hash"] != bci["bestblockhash"]: + bestheader = json.loads(args.bcli("getblockheader", bci["bestblockhash"])) + + if lastheader is None: + lastheader = bestheader["hash"] + elif bestheader["hash"] != lastheader: + next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson) + next_delta += bestheader["time"] - time.time() + next_is_mine = next_block_is_mine(bestheader["hash"], my_blocks) + logging.info("Received new block at height %d; next in %s (%s)", bestheader["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + lastheader = bestheader["hash"] + + # when is the next block due to be mined? + now = time.time() + if args.set_block_time is not None: + logging.debug("Setting start time to %d", args.set_block_time) + mine_time = args.set_block_time + action_time = now + is_mine = True + elif bestheader["height"] == 0: + logging.error("When mining first block in a new signet, must specify --set-block-time") + return 1 + else: + + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + mine_time = bestheader["time"] + time_delta + + is_mine = next_block_is_mine(bci["bestblockhash"], my_blocks) + + action_time = mine_time + if not is_mine: + action_time += args.backup_delay + + if args.standby_delay > 0: + action_time += args.standby_delay + elif mined_blocks == 0: + # for non-standby, always mine immediately on startup, + # even if the next block shouldn't be ours + action_time = now + + # don't want fractional times so round down + mine_time = int(mine_time) + action_time = int(action_time) + + # can't mine a block 2h in the future; 1h55m for some safety + action_time = max(action_time, mine_time - 6900) + + # ready to go? otherwise sleep and check for new block + if now < action_time: + sleep_for = min(action_time - now, 60) + if mine_time < now: + # someone else might have mined the block, + # so check frequently, so we don't end up late + # mining the next block if it's ours + sleep_for = min(20, sleep_for) + minestr = "mine" if is_mine else "backup" + logging.debug("Sleeping for %s, next block due in %s (%s)" % (seconds_to_hms(sleep_for), seconds_to_hms(mine_time - now), minestr)) + time.sleep(sleep_for) + continue + + # gbt + tmpl = json.loads(args.bcli("getblocktemplate", '{"rules":["signet","segwit"]}')) + if tmpl["previousblockhash"] != bci["bestblockhash"]: + logging.warning("GBT based off unexpected block (%s not %s), retrying", tmpl["previousblockhash"], bci["bestblockhash"]) + time.sleep(1) + continue + + logging.debug("GBT template: %s", tmpl) + + if tmpl["mintime"] > mine_time: + logging.info("Updating block time from %d to %d", mine_time, tmpl["mintime"]) + mine_time = tmpl["mintime"] + if mine_time > now: + logging.error("GBT mintime is in the future: %d is %d seconds later than %d", mine_time, (mine_time-now), now) + return 1 + + # address for reward + reward_addr, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + + # mine block + logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) + mined_blocks += 1 + psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + if not psbt_signed.get("complete",False): + logging.debug("Generated PSBT: %s" % (psbt,)) + sys.stderr.write("PSBT signing failed") + return 1 + block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) + block = finish_block(block, signet_solution, args.grind_cmd) + + # submit block + r = args.bcli("-stdin", "submitblock", input=ToHex(block).encode('utf8')) + + # report + bstr = "block" if is_mine else "backup block" + + next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson) + next_delta += block.nTime - time.time() + next_is_mine = next_block_is_mine(block.hash, my_blocks) + + logging.debug("Block hash %s payout to %s", block.hash, reward_addr) + logging.info("Mined %s at height %d; next in %s (%s)", bstr, tmpl["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + if r != "": + logging.warning("submitblock returned %s for height %d hash %s", r, tmpl["height"], block.hash) + lastheader = block.hash + +def do_calibrate(args): + if args.nbits is not None and args.seconds is not None: + sys.stderr.write("Can only specify one of --nbits or --seconds\n") + return 1 + if args.nbits is not None and len(args.nbits) != 8: + sys.stderr.write("Must specify 8 hex digits for --nbits") + return 1 + + TRIALS = 600 # gets variance down pretty low + TRIAL_BITS = 0x1e3ea75f # takes about 5m to do 600 trials + #TRIAL_BITS = 0x1e7ea75f # XXX + + header = CBlockHeader() + header.nBits = TRIAL_BITS + targ = nbits_to_target(header.nBits) + + start = time.time() + count = 0 + #CHECKS=[] + for i in range(TRIALS): + header.nTime = i + header.nNonce = 0 + headhex = header.serialize().hex() + cmd = args.grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + #newhead = FromHex(CBlockHeader(), newheadhex.decode('utf8')) + #count += newhead.nNonce + #if (i+1) % 100 == 0: + # CHECKS.append((i+1, count, time.time()-start)) + + #print("checks =", [c*1.0 / (b*targ*2**-256) for _,b,c in CHECKS]) + + avg = (time.time() - start) * 1.0 / TRIALS + #exp_count = 2**256 / targ * TRIALS + #print("avg =", avg, "count =", count, "exp_count =", exp_count) + + if args.nbits is not None: + want_targ = nbits_to_target(int(args.nbits,16)) + want_time = avg*targ/want_targ + else: + want_time = args.seconds if args.seconds is not None else 25 + want_targ = int(targ*(avg/want_time)) + + print("nbits=%08x for %ds average mining time" % (target_to_nbits(want_targ), want_time)) + return 0 + +def bitcoin_cli(basecmd, args, **kwargs): + cmd = basecmd + ["-signet"] + args + logging.debug("Calling bitcoin-cli: %r", cmd) + out = subprocess.run(cmd, stdout=subprocess.PIPE, **kwargs, check=True).stdout + if isinstance(out, bytes): + out = out.decode('utf8') + return out.strip() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cli", default="bitcoin-cli", type=str, help="bitcoin-cli command") + parser.add_argument("--debug", action="store_true", help="Print debugging info") + parser.add_argument("--quiet", action="store_true", help="Only print warnings/errors") + + cmds = parser.add_subparsers(help="sub-commands") + genpsbt = cmds.add_parser("genpsbt", help="Generate a block PSBT for signing") + genpsbt.set_defaults(fn=do_genpsbt) + + solvepsbt = cmds.add_parser("solvepsbt", help="Solve a signed block PSBT") + solvepsbt.set_defaults(fn=do_solvepsbt) + + generate = cmds.add_parser("generate", help="Mine blocks") + generate.set_defaults(fn=do_generate) + generate.add_argument("--ongoing", action="store_true", help="Keep mining blocks") + generate.add_argument("--max-blocks", default=None, type=int, help="Max blocks to mine (default=1)") + generate.add_argument("--set-block-time", default=None, type=int, help="Set block time (unix timestamp)") + generate.add_argument("--nbits", default=None, type=str, help="Target nBits (specify difficulty)") + generate.add_argument("--min-nbits", action="store_true", help="Target minimum nBits (use min difficulty)") + generate.add_argument("--poisson", action="store_true", help="Simulate randomised block times") + #generate.add_argument("--signcmd", default=None, type=str, help="Alternative signing command") + generate.add_argument("--multiminer", default=None, type=str, help="Specify which set of blocks to mine (eg: 1-40/100 for the first 40%%, 2/3 for the second 3rd)") + generate.add_argument("--backup-delay", default=300, type=int, help="Seconds to delay before mining blocks reserved for other miners (default=300)") + generate.add_argument("--standby-delay", default=0, type=int, help="Seconds to delay before mining blocks (default=0)") + + calibrate = cmds.add_parser("calibrate", help="Calibrate difficulty") + calibrate.set_defaults(fn=do_calibrate) + calibrate.add_argument("--nbits", type=str, default=None) + calibrate.add_argument("--seconds", type=int, default=None) + + for sp in [genpsbt, generate]: + sp.add_argument("--address", default=None, type=str, help="Address for block reward payment") + sp.add_argument("--descriptor", default=None, type=str, help="Descriptor for block reward payment") + + for sp in [solvepsbt, generate, calibrate]: + sp.add_argument("--grind-cmd", default=None, type=str, help="Command to grind a block header for proof-of-work") + + args = parser.parse_args(sys.argv[1:]) + + args.bcli = lambda *a, input=b"", **kwargs: bitcoin_cli(args.cli.split(" "), list(a), input=input, **kwargs) + + if hasattr(args, "address") and hasattr(args, "descriptor"): + if args.address is None and args.descriptor is None: + sys.stderr.write("Must specify --address or --descriptor\n") + return 1 + elif args.address is not None and args.descriptor is not None: + sys.stderr.write("Only specify one of --address or --descriptor\n") + return 1 + args.derived_addresses = {} + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + elif args.quiet: + logging.getLogger().setLevel(logging.WARNING) + else: + logging.getLogger().setLevel(logging.INFO) + + if hasattr(args, "fn"): + return args.fn(args) + else: + logging.error("Must specify command") + return 1 + +if __name__ == "__main__": + main() + + diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 4209fdb364..c50d4bef71 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -21,21 +21,21 @@ The script returns 0 if everything passes the checks. It returns 1 if either the ```sh -./verify.sh bitcoin-core-0.11.2 -./verify.sh bitcoin-core-0.12.0 -./verify.sh bitcoin-core-0.13.0-rc3 +./verify.py bitcoin-core-0.11.2 +./verify.py bitcoin-core-0.12.0 +./verify.py bitcoin-core-0.13.0-rc3 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./verify.sh bitcoin-core-0.11.2-osx -./verify.sh 0.12.0-linux -./verify.sh bitcoin-core-0.13.0-rc3-win64 +./verify.py bitcoin-core-0.11.2-osx +./verify.py 0.12.0-linux +./verify.py bitcoin-core-0.13.0-rc3-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./verify.sh bitcoin-core-0.13.0 delete +./verify.py bitcoin-core-0.13.0 delete ``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py new file mode 100755 index 0000000000..6cbaf2dc0c --- /dev/null +++ b/contrib/verifybinaries/verify.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitoin Core release binaries + +This script attempts to download the signature file SHA256SUMS.asc from +bitcoincore.org and bitcoin.org and compares them. +It first checks if the signature passes, and then downloads the files +specified in the file, and checks if the hashes of these files match those +that are specified in the signature file. +The script returns 0 if everything passes the checks. It returns 1 if either +the signature check or the hash check doesn't pass. If an error occurs the +return value is >= 2. +""" +from hashlib import sha256 +import os +import subprocess +import sys +from textwrap import indent + +WORKINGDIR = "/tmp/bitcoin_verify_binaries" +HASHFILE = "hashes.tmp" +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SIGNATUREFILENAME = "SHA256SUMS.asc" + + +def parse_version_string(version_str): + if version_str.startswith(VERSIONPREFIX): # remove version prefix + version_str = version_str[len(VERSIONPREFIX):] + + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "<version>-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "<version>-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file=None): + if local_file: + wget_args = ['wget', '-O', local_file, remote_file] + else: + # use timestamping mechanism if local filename is not explicitly set + wget_args = ['wget', '-N', remote_file] + + result = subprocess.run(wget_args, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + return contents1 == contents2 + + +def verify_with_gpg(signature_filename, output_filename): + result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', + output_filename, signature_filename], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode, result.stdout.decode().rstrip() + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +def main(args): + # sanity check + if len(args) < 1: + print("Error: need to specify a version on the command line") + return 3 + + # determine remote dir dependent on provided version string + version_base, version_rc, os_filter = parse_version_string(args[0]) + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigfile = remote_dir + SIGNATUREFILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + # fetch first signature file + sigfile1 = SIGNATUREFILENAME + success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) + if not success: + print("Error: couldn't fetch signature file. " + "Have you specified the version number in the following format?") + print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] " + f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") + print("wget output:") + print(indent(output, '\t')) + return 4 + + # fetch second signature file + sigfile2 = SIGNATUREFILENAME + ".2" + success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) + if not success: + print("bitcoin.org failed to provide signature file, " + "but bitcoincore.org did?") + print("wget output:") + print(indent(output, '\t')) + remove_files([sigfile1]) + return 5 + + # ensure that both signature files are equal + if not files_are_equal(sigfile1, sigfile2): + print("bitcoin.org and bitcoincore.org signature files were not equal?") + print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") + return 6 + + # check signature and extract data into file + retval, output = verify_with_gpg(sigfile1, HASHFILE) + if retval != 0: + if retval == 1: + print("Bad signature.") + elif retval == 2: + print("gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + print("gpg output:") + print(indent(output, '\t')) + remove_files([sigfile1, sigfile2, HASHFILE]) + return 1 + + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: "<hash> <binary_filename>" + with open(HASHFILE, 'r', encoding='utf8') as hash_file: + hashes_to_verify = [ + line.split()[:2] for line in hash_file if os_filter in line] + remove_files([HASHFILE]) + if not hashes_to_verify: + print("error: no files matched the platform specified") + return 7 + + # download binaries + for _, binary_filename in hashes_to_verify: + print(f"Downloading {binary_filename}") + download_with_wget(HOST1 + remote_dir + binary_filename) + + # verify hashes + offending_files = [] + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + if offending_files: + print("Hashes don't match.") + print("Offending files:") + print('\n'.join(offending_files)) + return 1 + verified_binaries = [entry[1] for entry in hashes_to_verify] + + # clean up files if desired + if len(args) >= 2: + print("Clean up the binaries") + remove_files([sigfile1, sigfile2] + verified_binaries) + else: + print(f"Keep the binaries in {WORKINGDIR}") + + print("Verified hashes of") + print('\n'.join(verified_binaries)) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh deleted file mode 100755 index 4296998631..0000000000 --- a/contrib/verifybinaries/verify.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -### This script attempts to download the signature file SHA256SUMS.asc from -### bitcoincore.org and bitcoin.org and compares them. -### It first checks if the signature passes, and then downloads the files specified in -### the file, and checks if the hashes of these files match those that are specified -### in the signature file. -### The script returns 0 if everything passes the checks. It returns 1 if either the -### signature check or the hash check doesn't pass. If an error occurs the return value is 2 - -export LC_ALL=C -function clean_up { - for file in "$@" - do - rm "$file" 2> /dev/null - done -} - -WORKINGDIR="/tmp/bitcoin_verify_binaries" -TMPFILE="hashes.tmp" - -SIGNATUREFILENAME="SHA256SUMS.asc" -RCSUBDIR="test" -HOST1="https://bitcoincore.org" -HOST2="https://bitcoin.org" -BASEDIR="/bin/" -VERSIONPREFIX="bitcoin-core-" -RCVERSIONSTRING="rc" - -if [ ! -d "$WORKINGDIR" ]; then - mkdir "$WORKINGDIR" -fi - -cd "$WORKINGDIR" || exit 1 - -#test if a version number has been passed as an argument -if [ -n "$1" ]; then - #let's also check if the version number includes the prefix 'bitcoin-', - # and add this prefix if it doesn't - if [[ $1 == "$VERSIONPREFIX"* ]]; then - VERSION="$1" - else - VERSION="$VERSIONPREFIX$1" - fi - - STRIPPEDLAST="${VERSION%-*}" - - #now let's see if the version string contains "rc" or a platform name (e.g. "osx") - if [[ "$STRIPPEDLAST-" == "$VERSIONPREFIX" ]]; then - BASEDIR="$BASEDIR$VERSION/" - else - # let's examine the last part to see if it's rc and/or platform name - STRIPPEDNEXTTOLAST="${STRIPPEDLAST%-*}" - if [[ "$STRIPPEDNEXTTOLAST-" == "$VERSIONPREFIX" ]]; then - - LASTSUFFIX="${VERSION##*-}" - VERSION="$STRIPPEDLAST" - - if [[ $LASTSUFFIX == *"$RCVERSIONSTRING"* ]]; then - RCVERSION="$LASTSUFFIX" - else - PLATFORM="$LASTSUFFIX" - fi - - else - RCVERSION="${STRIPPEDLAST##*-}" - PLATFORM="${VERSION##*-}" - - VERSION="$STRIPPEDNEXTTOLAST" - fi - - BASEDIR="$BASEDIR$VERSION/" - if [[ $RCVERSION == *"$RCVERSIONSTRING"* ]]; then - BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/" - fi - fi -else - echo "Error: need to specify a version on the command line" - exit 2 -fi - -if ! WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" - # shellcheck disable=SC1087 - echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - exit 2 -fi - -if ! WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME - exit 3 -fi - -SIGFILEDIFFS="$(diff $SIGNATUREFILENAME $SIGNATUREFILENAME.2)" -if [ "$SIGFILEDIFFS" != "" ]; then - echo "bitcoin.org and bitcoincore.org signature files were not equal?" - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 - exit 4 -fi - -#then we check it -GPGOUT=$(gpg --yes --decrypt --output "$TMPFILE" "$SIGNATUREFILENAME" 2>&1) - -#return value 0: good signature -#return value 1: bad signature -#return value 2: gpg error - -RET="$?" -if [ $RET -ne 0 ]; then - if [ $RET -eq 1 ]; then - #and notify the user if it's bad - echo "Bad signature." - elif [ $RET -eq 2 ]; then - #or if a gpg error has occurred - echo "gpg error. Do you have the Bitcoin Core binary release signing key installed?" - fi - - echo "gpg output:" - # shellcheck disable=SC2001 - echo "$GPGOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE - exit "$RET" -fi - -if [ -n "$PLATFORM" ]; then - grep $PLATFORM $TMPFILE > "$TMPFILE-plat" - TMPFILESIZE=$(stat -c%s "$TMPFILE-plat") - if [ $TMPFILESIZE -eq 0 ]; then - echo "error: no files matched the platform specified" && exit 3 - fi - mv "$TMPFILE-plat" $TMPFILE -fi - -#here we extract the filenames from the signature file -FILES=$(awk '{print $2}' "$TMPFILE") - -#and download these one by one -for file in $FILES -do - echo "Downloading $file" - wget --quiet -N "$HOST1$BASEDIR$file" -done - -#check hashes -DIFF=$(diff <(sha256sum $FILES) "$TMPFILE") - -if [ $? -eq 1 ]; then - echo "Hashes don't match." - echo "Offending files:" - echo "$DIFF"|grep "^<"|awk '{print "\t"$3}' - exit 1 -elif [ $? -gt 1 ]; then - echo "Error executing 'diff'" - exit 2 -fi - -if [ -n "$2" ]; then - echo "Clean up the binaries" - clean_up $FILES $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE -else - echo "Keep the binaries in $WORKINGDIR" - clean_up $TMPFILE -fi - -echo -e "Verified hashes of \n$FILES" - -exit 0 diff --git a/depends/Makefile b/depends/Makefile index 596a46d4a2..4cd4d72fc2 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -2,7 +2,7 @@ # Pattern rule to print variables, e.g. make print-top_srcdir print-%: - @echo $* = $($*) + @echo '$*' = '$($*)' # When invoking a sub-make, keep only the command line variable definitions # matching the pattern in the filter function. @@ -112,19 +112,27 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk +full_env=$(shell printenv) + build_id_string:=$(BUILD_ID_SALT) -build_id_string+=$(shell $(build_CC) --version 2>/dev/null) -build_id_string+=$(shell $(build_AR) --version 2>/dev/null) -build_id_string+=$(shell $(build_CXX) --version 2>/dev/null) -build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) -build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) + +# GCC only prints COLLECT_LTO_WRAPPER when invoked with just "-v", but we want +# the information from "-v -E -" as well, so just include both. +# +# '3>&1 1>&2 2>&3 > /dev/null' is supposed to swap stdin and stdout and silence +# stdin, since we only want the stderr output +build_id_string+=$(shell $(build_CC) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(build_CC) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) +build_id_string+=$(shell $(build_AR) --version 2>/dev/null) $(filter AR_%,$(full_env)) ZERO_AR_DATE=$(ZERO_AR_DATE) +build_id_string+=$(shell $(build_CXX) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(build_CXX) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) +build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) $(filter RANLIB_%,$(full_env)) +build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) $(filter STRIP_%,$(full_env)) $(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(host_CC) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) $(filter AR_%,$(full_env)) ZERO_AR_DATE=$(ZERO_AR_DATE) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(host_CXX) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(filter RANLIB_%,$(full_env)) +$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) $(filter STRIP_%,$(full_env)) ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) # Make sure that cache is invalidated when switching between system and @@ -133,6 +141,9 @@ build_id_string+=system_clang $(host_arch)_$(host_os)_id_string+=system_clang endif +build_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) +$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) + qrencode_packages_$(NO_QR) = $(qrencode_packages) qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) @@ -265,7 +276,7 @@ install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin14 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: diff --git a/depends/README.md b/depends/README.md index 9bc9b6714b..9e9878c595 100644 --- a/depends/README.md +++ b/depends/README.md @@ -44,7 +44,7 @@ The paths are automatically configured and no other options are needed unless ta #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5 + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5 xorriso #### For Win64 cross compilation diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ff8a252db9..29a3efdfe6 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -22,7 +22,7 @@ $(package)_toolset_$(host_os)=clang else $(package)_toolset_$(host_os)=gcc endif -$(package)_config_libraries=filesystem,system,thread,test +$(package)_config_libraries=filesystem,system,test $(package)_cxxflags=-std=c++17 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 2f79168212..21bf587eaf 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -2073,7 +2073,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = HAVE_BOOST_PROCESS +PREDEFINED = ENABLE_EXTERNAL_SIGNER # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index c66e79af71..12807bfb86 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -88,13 +88,14 @@ RPC interface will be abused. - **Secure string handling:** The RPC interface does not guarantee any escaping of data beyond what's necessary to encode it as JSON, although it does usually provide serialized data using a hex - representation of the bytes. If you use RPC data in your programs or - provide its data to other programs, you must ensure any problem - strings are properly escaped. For example, multiple websites have - been manipulated because they displayed decoded hex strings that - included HTML `<script>` tags. For this reason, and other - non-security reasons, it is recommended to display all serialized data - in hex form only. + representation of the bytes. If you use RPC data in your programs or + provide its data to other programs, you must ensure any problem strings + are properly escaped. For example, the `createwallet` RPC accepts + arguments such as `wallet_name` which is a string and could be used + for a path traversal attack without application level checks. Multiple + websites have been manipulated because they displayed decoded hex strings + that included HTML `<script>` tags. For this reason, and others, it is + recommended to display all serialized data in hex form only. ## RPC consistency guarantees diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 3b127703b7..ea06952af4 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -4,7 +4,7 @@ Unauthenticated REST Interface The REST API can be enabled with the `-rest` option. The interface runs on the same port as the JSON-RPC interface, by default port 8332 for mainnet, port 18332 for testnet, -and port 18443 for regtest. +port 38332 for signet, and port 18443 for regtest. REST Interface consistency guarantees ------------------------------------- @@ -62,7 +62,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name (main, test, regtest) +* chain : (string) current network name (main, test, signet, regtest) * blocks : (numeric) the current number of blocks processed in the server * headers : (numeric) the current number of headers we have validated * bestblockhash : (string) the hash of the currently best block @@ -111,12 +111,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 Returns various information about the TX mempool. Only supports JSON as output format. -* loaded : (boolean) if the mempool is fully loaded -* size : (numeric) the number of transactions in the TX mempool -* bytes : (numeric) size of the TX mempool in bytes -* usage : (numeric) total TX mempool memory usage -* maxmempool : (numeric) maximum memory usage for the mempool in bytes -* mempoolminfee : (numeric) minimum feerate (BTC per KB) for tx to be accepted +Refer to the `getmempoolinfo` RPC for documentation of the fields. `GET /rest/mempool/contents.json` diff --git a/doc/bips.md b/doc/bips.md index 8c20533c9b..a5e9a6c020 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -15,6 +15,9 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.21.0**): * [`BIP 35`](https://github.com/bitcoin/bips/blob/master/bip-0035.mediawiki): The 'mempool' protocol message (and the protocol version bump to 60002) has been implemented since **v0.7.0** ([PR #1641](https://github.com/bitcoin/bitcoin/pull/1641)). As of **v0.13.0**, this is only available for `NODE_BLOOM` (BIP 111) peers. * [`BIP 37`](https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki): The bloom filtering for transaction relaying, partial Merkle trees for blocks, and the protocol version bump to 70001 (enabling low-bandwidth SPV clients) has been implemented since **v0.8.0** ([PR #1795](https://github.com/bitcoin/bitcoin/pull/1795)). Disabled by default since **v0.19.0**, can be enabled by the `-peerbloomfilters` option. * [`BIP 42`](https://github.com/bitcoin/bips/blob/master/bip-0042.mediawiki): The bug that would have caused the subsidy schedule to resume after block 13440000 was fixed in **v0.9.2** ([PR #3842](https://github.com/bitcoin/bitcoin/pull/3842)). +* [`BIP 43`](https://github.com/bitcoin/bips/blob/master/bip-0043.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 43. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) +* [`BIP 44`](https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 44. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) +* [`BIP 49`](https://github.com/bitcoin/bips/blob/master/bip-0049.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 49. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) * [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). Starting **v0.17.0**, whether to send reject messages can be configured with the `-enablebip61` option, and support is deprecated (disabled by default) as of **v0.18.0**. Support was removed in **v0.20.0** ([PR #15437](https://github.com/bitcoin/bitcoin/pull/15437)). * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). @@ -24,6 +27,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.21.0**): Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). +* [`BIP 84`](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 84. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). diff --git a/doc/build-osx.md b/doc/build-osx.md index b46bf61c4b..04ee43f81d 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -30,6 +30,11 @@ If you want to build the disk image with `make deploy` (.dmg / optional), you ne brew install librsvg ``` +and [`macdeployqtplus`](../contrib/macdeploy/README.md) dependencies: +```shell +pip3 install ds_store mac_alias +``` + The wallet support requires one or both of the dependencies ([*SQLite*](#sqlite) and [*Berkeley DB*](#berkeley-db)) in the sections below. To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode). diff --git a/doc/build-unix.md b/doc/build-unix.md index 5c24886dbf..07fb9c453e 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -9,7 +9,7 @@ Note Always use absolute paths to configure and compile Bitcoin Core and the dependencies. For example, when specifying the path of the dependency: - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX + ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures the usage of the absolute path. @@ -82,7 +82,7 @@ Build requirements: Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: - sudo apt-get install libevent-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev + sudo apt-get install libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev BerkeleyDB is required for the wallet. @@ -166,9 +166,9 @@ miniupnpc https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and turned off by default. See the configure options for UPnP behavior desired: - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime + --without-miniupnpc No UPnP support, miniupnp not required + --disable-upnp-default (the default) UPnP support turned off by default at runtime + --enable-upnp-default UPnP support turned on by default at runtime libnatpmp --------- @@ -177,9 +177,9 @@ libnatpmp from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and turned off by default. See the configure options for NAT-PMP behavior desired: - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime + --without-natpmp No NAT-PMP support, libnatpmp not required + --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime + --enable-natpmp-default NAT-PMP support turned on by default at runtime Berkeley DB ----------- @@ -199,9 +199,9 @@ Boost ----- If you need to build Boost yourself: - sudo su - ./bootstrap.sh - ./bjam install + sudo su + ./bootstrap.sh + ./bjam install Security @@ -212,8 +212,8 @@ This can be disabled with: Hardening Flags: - ./configure --enable-hardening - ./configure --disable-hardening + ./configure --enable-hardening + ./configure --disable-hardening Hardening enables the following features: @@ -228,7 +228,7 @@ Hardening enables the following features: To test that you have built PIE executable, install scanelf, part of paxutils, and use: - scanelf -e ./bitcoin + scanelf -e ./bitcoin The output should contain: @@ -245,8 +245,8 @@ Hardening enables the following features: `scanelf -e ./bitcoin` The output should contain: - STK/REL/PTL - RW- R-- RW- + STK/REL/PTL + RW- R-- RW- The STK RW- means that the stack is readable and writeable but not executable. diff --git a/doc/build-windows.md b/doc/build-windows.md index 28b6aceb3c..d1b84eef42 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -103,7 +103,7 @@ Build using: cd depends make HOST=x86_64-w64-mingw32 cd .. - ./autogen.sh # not required when building from tarball + ./autogen.sh CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make sudo bash -c "echo 1 > /proc/sys/fs/binfmt_misc/status" # Enable WSL support for Win32 applications. diff --git a/doc/dependencies.md b/doc/dependencies.md index 6b2b904632..159c6bcc28 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -21,7 +21,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.6](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | +| Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | | SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | diff --git a/doc/descriptors.md b/doc/descriptors.md index 63acb9167f..c4fc2a66bf 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -191,7 +191,7 @@ steps, or for dumping wallet descriptors including private key material. In order to easily represent the sets of scripts currently supported by existing Bitcoin Core wallets, a convenience function `combo` is provided, which takes as input a public key, and describes a set of P2PK, -P2PKH, P2WPKH, and P2SH-P2WPH scripts for that key. In case the key is +P2PKH, P2WPKH, and P2SH-P2WPKH scripts for that key. In case the key is uncompressed, the set only includes P2PK and P2PKH scripts. ### Checksums diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 596f65cf10..8f2d7af089 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -75,6 +75,11 @@ tool to clean up patches automatically before submission. on the same line as the `if`, without braces. In every other case, braces are required, and the `then` and `else` clauses must appear correctly indented on a new line. + - There's no hard limit on line width, but prefer to keep lines to <100 + characters if doing so does not decrease readability. Break up long + function declarations over multiple lines using the Clang Format + [AlignAfterOpenBracket](https://clang.llvm.org/docs/ClangFormatStyleOptions.html) + style option. - **Symbol naming conventions**. These are preferred in new code, but are not required when doing so would need changes to significant pieces of existing @@ -780,6 +785,11 @@ Threads and synchronization get compile-time warnings about potential race conditions in code. Combine annotations in function declarations with run-time asserts in function definitions: + - In functions that are declared separately from where they are defined, the + thread safety annotations should be added exclusively to the function + declaration. Annotations on the definition could lead to false positives + (lack of compile failure) at call sites between the two. + ```C++ // txmempool.h class CTxMemPool diff --git a/doc/external-signer.md b/doc/external-signer.md new file mode 100644 index 0000000000..c91ea9bab7 --- /dev/null +++ b/doc/external-signer.md @@ -0,0 +1,171 @@ +# Support for signing transactions outside of Bitcoin Core + +Bitcoin Core can be launched with `-signer=<cmd>` where `<cmd>` is an external tool which can sign transactions and perform other functions. For example, it can be used to communicate with a hardware wallet. + +## Example usage + +The following example is based on the [HWI](https://github.com/bitcoin-core/HWI) tool. Although this tool is hosted under the Bitcoin Core GitHub organization and maintained by Bitcoin Core developers, it should be used with caution. It is considered experimental and has far less review than Bitcoin Core itself. Be particularly careful when running tools such as these on a computer with private keys on it. + +When using a hardware wallet, consult the manufacturer website for (alternative) software they recommend. As long as their software conforms to the standard below, it should be able to work with Bitcoin Core. + +Start Bitcoin Core: + +```sh +$ bitcoind -signer=../HWI/hwi.py +``` + +### Device setup + +Follow the hardware manufacturers instructions for the initial device setup, as well as their instructions for creating a backup. Alternatively, for some devices, you can use the `setup`, `restore` and `backup` commands provided by [HWI](https://github.com/bitcoin-core/HWI). + +### Create wallet and import keys + +Get a list of signing devices / services: + +``` +$ bitcoin-cli enumeratesigners +{ + "signers": [ + { + "fingerprint": "c8df832a" + } +] +``` + +The master key fingerprint is used to identify a device. + +Create a wallet, this automatically imports the public keys: + +```sh +$ bitcoin-cli createwallet "hww" true true "" true true true +``` + +### Verify an address + +Display an address on the device: + +```sh +$ bitcoin-cli -rpcwallet=<wallet> getnewaddress +$ bitcoin-cli -rpcwallet=<wallet> signerdisplayaddress <address> +``` + +Replace `<address>` with the result of `getnewaddress`. + +### Spending + +Under the hood this uses a [Partially Signed Bitcoin Transaction](psbt.md). + +```sh +$ bitcoin-cli -rpcwallet=<wallet> sendtoaddress <address> <amount> +``` + +This prompts your hardware wallet to sign, and fail if it's not connected. If successful +it automatically broadcasts the transaction. + +```sh +{"complete": true, "txid": <txid>} +``` + +## Signer API + +In order to be compatible with Bitcoin Core any signer command should conform to the specification below. This specification is subject to change. Ideally a BIP should propose a standard so that other wallets can also make use of it. + +Prerequisite knowledge: +* [Output Descriptors](descriptors.md) +* Partially Signed Bitcoin Transaction ([PSBT](psbt.md)) + +### `enumerate` (required) + +Usage: +``` +$ <cmd> enumerate +[ + { + "fingerprint": "00000000" + } +] +``` + +The command MUST return an (empty) array with at least a `fingerprint` field. + +A future extension could add an optional return field with device capabilities. Perhaps a descriptor with wildcards. For example: `["pkh("44'/0'/$'/{0,1}/*"), sh(wpkh("49'/0'/$'/{0,1}/*")), wpkh("84'/0'/$'/{0,1}/*")]`. This would indicate the device supports legacy, wrapped SegWit and native SegWit. In addition it restricts the derivation paths that can used for those, to maintain compatibility with other wallet software. It also indicates the device, or the driver, doesn't support multisig. + +A future extension could add an optional return field `reachable`, in case `<cmd>` knows a signer exists but can't currently reach it. + +### `signtransaction` (required) + +Usage: +``` +$ <cmd> --fingerprint=<fingerprint> (--testnet) signtransaction <psbt> +base64_encode_signed_psbt +``` + +The command returns a psbt with any signatures. + +The `psbt` SHOULD include bip32 derivations. The command SHOULD fail if none of the bip32 derivations match a key owned by the device. + +The command SHOULD fail if the user cancels. + +The command MAY complain if `--testnet` is set, but any of the BIP32 derivation paths contain a coin type other than `1h` (and vice versa). + +### `getdescriptors` (optional) + +Usage: + +``` +$ <cmd> --fingerprint=<fingerprint> (--testnet) getdescriptors <account> +<xpub> +``` + +Returns descriptors supported by the device. Example: + +``` +$ <cmd> --fingerprint=00000000 --testnet getdescriptors +{ + "receive": [ + "pkh([00000000/44h/0h/0h]xpub6C.../0/*)#fn95jwmg", + "sh(wpkh([00000000/49h/0h/0h]xpub6B..../0/*))#j4r9hntt", + "wpkh([00000000/84h/0h/0h]xpub6C.../0/*)#qw72dxa9" + ], + "internal": [ + "pkh([00000000/44h/0h/0h]xpub6C.../1/*)#c8q40mts", + "sh(wpkh([00000000/49h/0h/0h]xpub6B..../1/*))#85dn0v75", + "wpkh([00000000/84h/0h/0h]xpub6C..../1/*)#36mtsnda" + ] +} +``` + +### `displayaddress` (optional) + +Usage: +``` +<cmd> --fingerprint=<fingerprint> (--testnet) displayaddress --desc descriptor +``` + +Example, display the first native SegWit receive address on Testnet: + +``` +<cmd> --fingerprint=00000000 --testnet displayaddress --desc "wpkh([00000000/84h/1h/0h]tpubDDUZ..../0/0)" +``` + +The command MUST be able to figure out the address type from the descriptor. + +If <descriptor> contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device. + +If <descriptor> contains an xpub, the command MUST fail if it does not match the xpub known by the device. + +The command MAY complain if `--testnet` is set, but the BIP32 coin type is not `1h` (and vice versa). + +## How Bitcoin Core uses the Signer API + +The `enumeratesigners` RPC simply calls `<cmd> enumerate`. + +The `createwallet` RPC calls: + +* `<cmd> --fingerprint=00000000 getdescriptors 0` + +It then imports descriptors for all support address types, in a BIP44/49/84 compatible manner. + +The `displayaddress` RPC reuses some code from `getaddressinfo` on the provided address and obtains the inferred descriptor. It then calls `<cmd> --fingerprint=00000000 displayaddress --desc=<descriptor>`. + +`sendtoaddress` and `sendmany` check `inputs->bip32_derivs` to see if any inputs have the same `master_fingerprint` as the signer. If so, it calls `<cmd> --fingerprint=00000000 signtransaction <psbt>`. It waits for the device to return a (partially) signed psbt, tries to finalize it and broadcasts the transaction. diff --git a/doc/files.md b/doc/files.md index 545e8fc92c..a80adf30ea 100644 --- a/doc/files.md +++ b/doc/files.md @@ -64,6 +64,7 @@ Subdirectory | File(s) | Description `./` | `ip_asn.map` | IP addresses to Autonomous System Numbers (ASNs) mapping used for bucketing of the peers; path can be specified with the `-asmap` option `./` | `mempool.dat` | Dump of the mempool's transactions `./` | `onion_v3_private_key` | Cached Tor onion service private key for `-listenonion` option +`./` | `i2p_private_key` | Private key that corresponds to our I2P address. When `-i2psam=` is specified the contents of this file is used to identify ourselves for making outgoing connections to I2P peers and possibly accepting incoming ones. Automatically generated if it does not exist. `./` | `peers.dat` | Peer IP address database (custom format) `./` | `settings.json` | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option `./` | `.cookie` | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 80ce821091..87df2bbbb9 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -157,3 +157,77 @@ $ FUZZ=process_message honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/fuzz ``` Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. + +## Fuzzing the Bitcoin Core P2P layer using Honggfuzz NetDriver + +Honggfuzz NetDriver allows for very easy fuzzing of TCP servers such as Bitcoin +Core without having to write any custom fuzzing harness. The `bitcoind` server +process is largely fuzzed without modification. + +This makes the fuzzing highly realistic: a bug reachable by the fuzzer is likely +also remotely triggerable by an untrusted peer. + +To quickly get started fuzzing the P2P layer using Honggfuzz NetDriver: + +```sh +$ mkdir bitcoin-honggfuzz-p2p/ +$ cd bitcoin-honggfuzz-p2p/ +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ ./autogen.sh +$ git clone https://github.com/google/honggfuzz +$ cd honggfuzz/ +$ make +$ cd .. +$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang \ + CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ \ + ./configure --disable-wallet --with-gui=no \ + --with-sanitizers=address,undefined +$ git apply << "EOF" +diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp +index 455a82e39..2faa3f80f 100644 +--- a/src/bitcoind.cpp ++++ b/src/bitcoind.cpp +@@ -158,7 +158,11 @@ static bool AppInit(int argc, char* argv[]) + return fRet; + } + ++#ifdef HFND_FUZZING_ENTRY_FUNCTION_CXX ++HFND_FUZZING_ENTRY_FUNCTION_CXX(int argc, char* argv[]) ++#else + int main(int argc, char* argv[]) ++#endif + { + #ifdef WIN32 + util::WinCmdLineArgs winArgs; +diff --git a/src/net.cpp b/src/net.cpp +index cf987b699..636a4176a 100644 +--- a/src/net.cpp ++++ b/src/net.cpp +@@ -709,7 +709,7 @@ int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) + } + + // Check start string, network magic +- if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { ++ if (false && memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { // skip network magic checking + LogPrint(BCLog::NET, "HEADER ERROR - MESSAGESTART (%s, %u bytes), received %s, peer=%d\n", hdr.GetCommand(), hdr.nMessageSize, HexStr(hdr.pchMessageStart), m_node_id); + return -1; + } +@@ -768,7 +768,7 @@ Optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono::mic + RandAddEvent(ReadLE32(hash.begin())); + + // Check checksum and header command string +- if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { ++ if (false && memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { // skip checksum checking + LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s, peer=%d\n", + SanitizeString(msg->m_command), msg->m_message_size, + HexStr(Span<uint8_t>(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), +EOF +$ make -C src/ bitcoind +$ mkdir -p inputs/ +$ honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ + -E HFND_TCP_PORT=18444 -f inputs/ -- \ + src/bitcoind -regtest -discover=0 -dns=0 -dnsseed=0 -listenonion=0 \ + -nodebuglogfile -bind=127.0.0.1:18444 -logthreadnames \ + -debug +``` diff --git a/doc/guix.md b/doc/guix.md new file mode 100644 index 0000000000..fa9c74f486 --- /dev/null +++ b/doc/guix.md @@ -0,0 +1,3 @@ +# Bootstrappable Bitcoin Core Builds + +See [contrib/guix/README.md](../contrib/guix/README.md) diff --git a/doc/init.md b/doc/init.md index 99aa0a0def..399b819bf4 100644 --- a/doc/init.md +++ b/doc/init.md @@ -53,11 +53,12 @@ Paths All three configurations assume several paths that might need to be adjusted. -Binary: `/usr/bin/bitcoind` -Configuration file: `/etc/bitcoin/bitcoin.conf` -Data directory: `/var/lib/bitcoind` -PID file: `/var/run/bitcoind/bitcoind.pid` (OpenRC and Upstart) or `/run/bitcoind/bitcoind.pid` (systemd) -Lock file: `/var/lock/subsys/bitcoind` (CentOS) + Binary: /usr/bin/bitcoind + Configuration file: /etc/bitcoin/bitcoin.conf + Data directory: /var/lib/bitcoind + PID file: /var/run/bitcoind/bitcoind.pid (OpenRC and Upstart) or + /run/bitcoind/bitcoind.pid (systemd) + Lock file: /var/lock/subsys/bitcoind (CentOS) The PID directory (if applicable) and data directory should both be owned by the bitcoin user and group. It is advised for security reasons to make the @@ -83,10 +84,10 @@ OpenRC). ### macOS -Binary: `/usr/local/bin/bitcoind` -Configuration file: `~/Library/Application Support/Bitcoin/bitcoin.conf` -Data directory: `~/Library/Application Support/Bitcoin` -Lock file: `~/Library/Application Support/Bitcoin/.lock` + Binary: /usr/local/bin/bitcoind + Configuration file: ~/Library/Application Support/Bitcoin/bitcoin.conf + Data directory: ~/Library/Application Support/Bitcoin + Lock file: ~/Library/Application Support/Bitcoin/.lock Installing Service Configuration ----------------------------------- diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index edbc0911a1..8f890da532 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -16,6 +16,10 @@ if BUILD_BITCOIN_TX dist_man1_MANS+=bitcoin-tx.1 endif +if BUILD_BITCOIN_UTIL + dist_man1_MANS+=bitcoin-util.1 +endif + if ENABLE_WALLET if BUILD_BITCOIN_WALLET dist_man1_MANS+=bitcoin-wallet.1 diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 588ae81fce..6bcad7006b 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -2,4 +2,4 @@ .SH NAME bitcoin-cli \- manual page for bitcoin-cli -This is a placefolder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index 9c75e9fe54..ff4d1d2c7a 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -2,4 +2,4 @@ .SH NAME bitcoin-qt \- manual page for bitcoin-qt -This is a placefolder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index 148a5890b0..776bb46234 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -2,4 +2,4 @@ .SH NAME bitcoin-tx \- manual page for bitcoin-tx -This is a placefolder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-util.1 b/doc/man/bitcoin-util.1 new file mode 100644 index 0000000000..5c733c6e21 --- /dev/null +++ b/doc/man/bitcoin-util.1 @@ -0,0 +1,5 @@ +.TH BITCOIN-UTIL "1" +.SH NAME +bitcoin-util \- manual page for bitcoin-util + +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 index 69133b33f7..2da43dec66 100644 --- a/doc/man/bitcoin-wallet.1 +++ b/doc/man/bitcoin-wallet.1 @@ -2,4 +2,4 @@ .SH NAME bitcoin-wallet \- manual page for bitcoin-wallet -This is a placefolder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index de338182ff..2c88f74520 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -2,4 +2,4 @@ .SH NAME bitcoind \- manual page for bitcoind -This is a placefolder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/release-notes-18466.md b/doc/release-notes-18466.md new file mode 100644 index 0000000000..e46d5064a3 --- /dev/null +++ b/doc/release-notes-18466.md @@ -0,0 +1,10 @@ +Low-level RPC changes +--------------------- + +- Error codes have been updated to be more accurate for the following error cases (#18466): + - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature + is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). diff --git a/doc/release-notes.md b/doc/release-notes.md index 8f1e03e16b..0f248494c7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -93,6 +93,10 @@ Tools and Utilities Wallet ------ +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + GUI changes ----------- diff --git a/doc/release-notes/release-notes-0.21.0.md b/doc/release-notes/release-notes-0.21.0.md new file mode 100644 index 0000000000..3baba3d49b --- /dev/null +++ b/doc/release-notes/release-notes-0.21.0.md @@ -0,0 +1,1336 @@ +0.21.0 Release Notes +==================== + +Bitcoin Core version 0.21.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.21.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +The node's known peers are persisted to disk in a file called `peers.dat`. The +format of this file has been changed in a backwards-incompatible way in order to +accommodate the storage of Tor v3 and other BIP155 addresses. This means that if +the file is modified by 0.21.0 or newer then older versions will not be able to +read it. Those old versions, in the event of a downgrade, will log an error +message "Incorrect keysize in addrman deserialization" and will continue normal +operation as if the file was missing, creating a new empty one. (#19954, #20284) + +Notable changes +=============== + +P2P and network changes +----------------------- + +- The mempool now tracks whether transactions submitted via the wallet or RPCs + have been successfully broadcast. Every 10-15 minutes, the node will try to + announce unbroadcast transactions until a peer requests it via a `getdata` + message or the transaction is removed from the mempool for other reasons. + The node will not track the broadcast status of transactions submitted to the + node using P2P relay. This version reduces the initial broadcast guarantees + for wallet transactions submitted via P2P to a node running the wallet. (#18038) + +- The size of the set of transactions that peers have announced and we consider + for requests has been reduced from 100000 to 5000 (per peer), and further + announcements will be ignored when that limit is reached. If you need to dump + (very) large batches of transactions, exceptions can be made for trusted + peers using the "relay" network permission. For localhost for example it can + be enabled using the command line option `-whitelist=relay@127.0.0.1`. + (#19988) + +- This release adds support for Tor version 3 hidden services, and rumoring them + over the network to other peers using + [BIP155](https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki). + Version 2 hidden services are still fully supported by Bitcoin Core, but the + Tor network will start + [deprecating](https://blog.torproject.org/v2-deprecation-timeline) them in the + coming months. (#19954) + +- The Tor onion service that is automatically created by setting the + `-listenonion` configuration parameter will now be created as a Tor v3 service + instead of Tor v2. The private key that was used for Tor v2 (if any) will be + left untouched in the `onion_private_key` file in the data directory (see + `-datadir`) and can be removed if not needed. Bitcoin Core will no longer + attempt to read it. The private key for the Tor v3 service will be saved in a + file named `onion_v3_private_key`. To use the deprecated Tor v2 service (not + recommended), the `onion_private_key` can be copied over + `onion_v3_private_key`, e.g. + `cp -f onion_private_key onion_v3_private_key`. (#19954) + +- The client writes a file (`anchors.dat`) at shutdown with the network addresses + of the node’s two outbound block-relay-only peers (so called "anchors"). The + next time the node starts, it reads this file and attempts to reconnect to those + same two peers. This prevents an attacker from using node restarts to trigger a + complete change in peers, which would be something they could use as part of an + eclipse attack. (#17428) + +- This release adds support for serving + [BIP157](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) compact + filters to peers on the network when enabled using + `-blockfilterindex=1 -peerblockfilters=1`. (#16442) + +- This release adds support for signets + ([BIP325](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki)) in + addition to the existing mainnet, testnet, and regtest networks. Signets are + centrally-controlled test networks, allowing them to be more predictable + test environments than the older testnet. One public signet is maintained, and + selectable using `-signet`. It is also possible to create personal signets. + (#18267). + +- This release implements + [BIP339](https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki) + wtxid relay. When negotiated, transactions are announced using their wtxid + instead of their txid. (#18044). + +- This release implements the proposed Taproot consensus rules + ([BIP341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki) and + [BIP342](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki)), + without activation on mainnet. Experimentation with Taproot can be done on + signet, where its rules are already active. (#19553) + +Updated RPCs +------------ + +- The `getpeerinfo` RPC has a new `network` field that provides the type of + network ("ipv4", "ipv6", or "onion") that the peer connected through. (#20002) + +- The `getpeerinfo` RPC now has additional `last_block` and `last_transaction` + fields that return the UNIX epoch time of the last block and the last *valid* + transaction received from each peer. (#19731) + +- `getnetworkinfo` now returns two new fields, `connections_in` and + `connections_out`, that provide the number of inbound and outbound peer + connections. These new fields are in addition to the existing `connections` + field, which returns the total number of peer connections. (#19405) + +- Exposed transaction version numbers are now treated as unsigned 32-bit + integers instead of signed 32-bit integers. This matches their treatment in + consensus logic. Versions greater than 2 continue to be non-standard + (matching previous behavior of smaller than 1 or greater than 2 being + non-standard). Note that this includes the `joinpsbt` command, which combines + partially-signed transactions by selecting the highest version number. + (#16525) + +- `getmempoolinfo` now returns an additional `unbroadcastcount` field. The + mempool tracks locally submitted transactions until their initial broadcast + is acknowledged by a peer. This field returns the count of transactions + waiting for acknowledgement. + +- Mempool RPCs such as `getmempoolentry` and `getrawmempool` with + `verbose=true` now return an additional `unbroadcast` field. This indicates + whether initial broadcast of the transaction has been acknowledged by a + peer. `getmempoolancestors` and `getmempooldescendants` are also updated. + +- The `getpeerinfo` RPC no longer returns the `banscore` field unless the configuration + option `-deprecatedrpc=banscore` is used. The `banscore` field will be fully + removed in the next major release. (#19469) + +- The `testmempoolaccept` RPC returns `vsize` and a `fees` object with the `base` fee + if the transaction would pass validation. (#19940) + +- The `getpeerinfo` RPC now returns a `connection_type` field. This indicates + the type of connection established with the peer. It will return one of six + options. For more information, see the `getpeerinfo` help documentation. + (#19725) + +- The `getpeerinfo` RPC no longer returns the `addnode` field by default. This + field will be fully removed in the next major release. It can be accessed + with the configuration option `-deprecatedrpc=getpeerinfo_addnode`. However, + it is recommended to instead use the `connection_type` field (it will return + `manual` when addnode is true). (#19725) + +- The `getpeerinfo` RPC no longer returns the `whitelisted` field by default. + This field will be fully removed in the next major release. It can be accessed + with the configuration option `-deprecatedrpc=getpeerinfo_whitelisted`. However, + it is recommended to instead use the `permissions` field to understand if specific + privileges have been granted to the peer. (#19770) + +- The `walletcreatefundedpsbt` RPC call will now fail with + `Insufficient funds` when inputs are manually selected but are not enough to cover + the outputs and fee. Additional inputs can automatically be added through the + new `add_inputs` option. (#16377) + +- The `fundrawtransaction` RPC now supports `add_inputs` option that when `false` + prevents adding more inputs if necessary and consequently the RPC fails. + +Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. + +New RPCs +-------- + +- The `getindexinfo` RPC returns the actively running indices of the node, + including their current sync status and height. It also accepts an `index_name` + to specify returning the status of that index only. (#19550) + +Build System +------------ + +Updated settings +---------------- + +- The same ZeroMQ notification (e.g. `-zmqpubhashtx=address`) can now be + specified multiple times to publish the same notification to different ZeroMQ + sockets. (#18309) + +- The `-banscore` configuration option, which modified the default threshold for + disconnecting and discouraging misbehaving peers, has been removed as part of + changes in 0.20.1 and in this release to the handling of misbehaving peers. + Refer to "Changes regarding misbehaving peers" in the 0.20.1 release notes for + details. (#19464) + +- The `-debug=db` logging category, which was deprecated in 0.20 and replaced by + `-debug=walletdb` to distinguish it from `coindb`, has been removed. (#19202) + +- A `download` permission has been extracted from the `noban` permission. For + compatibility, `noban` implies the `download` permission, but this may change + in future releases. Refer to the help of the affected settings `-whitebind` + and `-whitelist` for more details. (#19191) + +- Netmasks that contain 1-bits after 0-bits (the 1-bits are not contiguous on + the left side, e.g. 255.0.255.255) are no longer accepted. They are invalid + according to RFC 4632. Netmasks are used in the `-rpcallowip` and `-whitelist` + configuration options and in the `setban` RPC. (#19628) + +- The `-blocksonly` setting now completely disables fee estimation. (#18766) + +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. + +Tools and Utilities +------------------- + +- A new `bitcoin-cli -netinfo` command provides a network peer connections + dashboard that displays data from the `getpeerinfo` and `getnetworkinfo` RPCs + in a human-readable format. An optional integer argument from `0` to `4` may + be passed to see increasing levels of detail. (#19643) + +- A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` + followed by `generatetoaddress`, can generate blocks for command line testing + purposes. This is a client-side version of the former `generate` RPC. See the + help for details. (#19133) + +- The `bitcoin-cli -getinfo` command now displays the wallet name and balance for + each of the loaded wallets when more than one is loaded (e.g. in multiwallet + mode) and a wallet is not specified with `-rpcwallet`. (#18594) + +- The `connections` field of `bitcoin-cli -getinfo` is now expanded to return a JSON + object with `in`, `out` and `total` numbers of peer connections. It previously + returned a single integer value for the total number of peer connections. (#19405) + +New settings +------------ + +- The `startupnotify` option is used to specify a command to + execute when Bitcoin Core has finished with its startup + sequence. (#15367) + +Wallet +------ + +- Backwards compatibility has been dropped for two `getaddressinfo` RPC + deprecations, as notified in the 0.20 release notes. The deprecated `label` + field has been removed as well as the deprecated `labels` behavior of + returning a JSON object containing `name` and `purpose` key-value pairs. Since + 0.20, the `labels` field returns a JSON array of label names. (#19200) + +- To improve wallet privacy, the frequency of wallet rebroadcast attempts is + reduced from approximately once every 15 minutes to once every 12-36 hours. + To maintain a similar level of guarantee for initial broadcast of wallet + transactions, the mempool tracks these transactions as a part of the newly + introduced unbroadcast set. See the "P2P and network changes" section for + more information on the unbroadcast set. (#18038) + +- The `sendtoaddress` and `sendmany` RPCs accept an optional `verbose=True` + argument to also return the fee reason about the sent tx. (#19501) + +- The wallet can create a transaction without change even when the keypool is + empty. Previously it failed. (#17219) + +- The `-salvagewallet` startup option has been removed. A new `salvage` command + has been added to the `bitcoin-wallet` tool which performs the salvage + operations that `-salvagewallet` did. (#18918) + +- A new configuration flag `-maxapsfee` has been added, which sets the max + allowed avoid partial spends (APS) fee. It defaults to 0 (i.e. fee is the + same with and without APS). Setting it to -1 will disable APS, unless + `-avoidpartialspends` is set. (#14582) + +- The wallet will now avoid partial spends (APS) by default, if this does not + result in a difference in fees compared to the non-APS variant. The allowed + fee threshold can be adjusted using the new `-maxapsfee` configuration + option. (#14582) + +- The `createwallet`, `loadwallet`, and `unloadwallet` RPCs now accept + `load_on_startup` options to modify the settings list. Unless these options + are explicitly set to true or false, the list is not modified, so the RPC + methods remain backwards compatible. (#15937) + +- A new `send` RPC with similar syntax to `walletcreatefundedpsbt`, including + support for coin selection and a custom fee rate, is added. The `send` RPC is + experimental and may change in subsequent releases. (#16378) + +- The `estimate_mode` parameter is now case-insensitive in the `bumpfee`, + `fundrawtransaction`, `sendmany`, `sendtoaddress`, `send` and + `walletcreatefundedpsbt` RPCs. (#11413) + +- The `bumpfee` RPC now uses `conf_target` rather than `confTarget` in the + options. (#11413) + +- `fundrawtransaction` and `walletcreatefundedpsbt` when used with the + `lockUnspents` argument now lock manually selected coins, in addition to + automatically selected coins. Note that locked coins are never used in + automatic coin selection, but can still be manually selected. (#18244) + +- The `-zapwallettxes` startup option has been removed and its functionality + removed from the wallet. This option was originally intended to allow for + rescuing wallets which were affected by a malleability attack. More recently, + it has been used in the fee bumping of transactions that did not signal RBF. + This functionality has been superseded with the abandon transaction feature. (#19671) + +- The error code when no wallet is loaded, but a wallet RPC is called, has been + changed from `-32601` (method not found) to `-18` (wallet not found). + (#20101) + +### Automatic wallet creation removed + +Bitcoin Core will no longer automatically create new wallets on startup. It will +load existing wallets specified by `-wallet` options on the command line or in +`bitcoin.conf` or `settings.json` files. And by default it will also load a +top-level unnamed ("") wallet. However, if specified wallets don't exist, +Bitcoin Core will now just log warnings instead of creating new wallets with +new keys and addresses like previous releases did. + +New wallets can be created through the GUI (which has a more prominent create +wallet option), through the `bitcoin-cli createwallet` or `bitcoin-wallet +create` commands, or the `createwallet` RPC. (#15454, #20186) + +### Experimental Descriptor Wallets + +Please note that Descriptor Wallets are still experimental and not all expected functionality +is available. Additionally there may be some bugs and current functions may change in the future. +Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). + +0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store +scriptPubKey information using output descriptors. This is in contrast to the Legacy Wallet +structure where keys are used to implicitly generate scriptPubKeys and addresses. Because of this +shift to being script based instead of key based, many of the confusing things that Legacy +Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition +of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. +Descriptor Wallets also uses different semantics for watch-only things and imports. + +As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. +Users who already have a Bitcoin Core wallet can continue to use it as they did before without +any change in behavior. Newly created Legacy Wallets (which remains the default type of wallet) will +behave as they did in previous versions of Bitcoin Core. + +The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing +things. They are intended to behave similarly except for the import/export and watchonly functionality +as described below. + +#### Creating Descriptor Wallets + +Descriptor wallets are not the default type of wallet. + +In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a +Descriptor Wallet should be created. And a `descriptors` option has been added to `createwallet` RPC. +Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. + +Without those options being set, a Legacy Wallet will be created instead. + +#### `IsMine` Semantics + +`IsMine` refers to the function used to determine whether a script belongs to the wallet. +This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets +returns true if the wallet would be able to sign an input that spends an output with that script. +Since keys can be involved in a variety of different scripts, this definition for `IsMine` can +lead to many unexpected scripts being considered part of the wallet. + +With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by +the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly +what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` +in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` +allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type +(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type +even without that address being requested from the wallet. Descriptor Wallets do not +allow for this and will only watch for the addresses that were explicitly requested from the wallet. + +These changes to `IsMine` will make it easier to reason about what scripts the wallet will +actually be watching for in outputs. However for the vast majority of users, this change is +largely transparent and will not have noticeable effect. + +#### Imports and Exports + +In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts +and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` +logic as it has to distinguish between spendable and watchonly. + +Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be +imported. These descriptors are then added to the wallet as if it were a descriptor generated by +the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish +between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also +different and described in more detail in the next section. + +To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax +similar to that of `importmulti`. + +As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys +the existing import RPCs have been disabled for descriptor wallets. +New export RPCs for Descriptor Wallets have not yet been added. + +The following RPCs are disabled for Descriptor Wallets: + +* `importprivkey` +* `importpubkey` +* `importaddress` +* `importwallet` +* `dumpprivkey` +* `dumpwallet` +* `importmulti` +* `addmultisigaddress` +* `sethdseed` + +#### Watchonly Wallets + +A Legacy Wallet contains both private keys and scripts that were being watched. +Those watched scripts would not contribute to your normal balance. In order to see the watchonly +balance and to use watchonly things in transactions, an `include_watchonly` option was added +to many RPCs that would allow users to do that. However it is easy to forget to include this option. + +Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be +watchonly depending on whether it was created with private keys disabled. This eliminates the need +to distinguish between things that are watchonly and things that are not within a wallet itself. + +This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has +a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), +then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT +workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just +non-functional. + +This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such +multiple key descriptors as some transactions could be signed and broadcast and others not. This is +due to some transactions containing only single key inputs, while others would contain both single +key and multiple key inputs, depending on which are available and how the coin selection algorithm +selects inputs. However this is not considered to be a supported use case; multisigs +should be in their own wallets which do not already have descriptors. Although users cannot export +descriptors with private keys for now as explained earlier. + +#### BIP 44/49/84 Support + +The change to using descriptors changes the default derivation paths used by Bitcoin Core +to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without +issue. + +#### SQLite Database Backend + +Descriptor wallets use SQLite for the wallet file instead of the Berkeley DB used in legacy wallets. +This will break compatibility with any existing tooling that operates on wallets, however compatibility +was already being broken by the move to descriptors. + +### Wallet RPC changes + +- The `upgradewallet` RPC replaces the `-upgradewallet` command line option. + (#15761) + +- The `settxfee` RPC will fail if the fee was set higher than the `-maxtxfee` + command line setting. The wallet will already fail to create transactions + with fees higher than `-maxtxfee`. (#18467) + +- A new `fee_rate` parameter/option denominated in satoshis per vbyte (sat/vB) + is introduced to the `sendtoaddress`, `sendmany`, `fundrawtransaction` and + `walletcreatefundedpsbt` RPCs as well as to the experimental new `send` + RPC. The legacy `feeRate` option in `fundrawtransaction` and + `walletcreatefundedpsbt` still exists for setting a fee rate in BTC per 1,000 + vbytes (BTC/kvB), but it is expected to be deprecated soon to avoid + confusion. For these RPCs, the fee rate error message is updated from BTC/kB + to sat/vB and the help documentation in BTC/kB is updated to BTC/kvB. The + `send` and `sendtoaddress` RPC examples are updated to aid users in creating + transactions with explicit fee rates. (#20305, #11413) + +- The `bumpfee` RPC `fee_rate` option is changed from BTC/kvB to sat/vB and the + help documentation is updated. Users are warned that this is a breaking API + change, but it should be relatively benign: the large (100,000 times) + difference between BTC/kvB and sat/vB units means that a transaction with a + fee rate mistakenly calculated in BTC/kvB rather than sat/vB should raise an + error due to the fee rate being set too low. In the worst case, the + transaction may send at 1 sat/vB, but as Replace-by-Fee (BIP125 RBF) is active + by default when an explicit fee rate is used, the transaction fee can be + bumped. (#20305) + +GUI changes +----------- + +- Wallets created or loaded in the GUI will now be automatically loaded on + startup, so they don't need to be manually reloaded next time Bitcoin Core is + started. The list of wallets to load on startup is stored in + `\<datadir\>/settings.json` and augments any command line or `bitcoin.conf` + `-wallet=` settings that specify more wallets to load. Wallets that are + unloaded in the GUI get removed from the settings list so they won't load + again automatically next startup. (#19754) + +- The GUI Peers window no longer displays a "Ban Score" field. This is part of + changes in 0.20.1 and in this release to the handling of misbehaving + peers. Refer to "Changes regarding misbehaving peers" in the 0.20.1 release + notes for details. (#19512) + +Low-level changes +================= + +RPC +--- + +- To make RPC `sendtoaddress` more consistent with `sendmany` the following error + `sendtoaddress` codes were changed from `-4` to `-6`: + - Insufficient funds + - Fee estimation failed + - Transaction has too long of a mempool chain + +- The `sendrawtransaction` error code for exceeding `maxfeerate` has been changed from + `-26` to `-25`. The error string has been changed from "absurdly-high-fee" to + "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)." The + `testmempoolaccept` RPC returns `max-fee-exceeded` rather than `absurdly-high-fee` + as the `reject-reason`. (#19339) + +- To make wallet and rawtransaction RPCs more consistent, the error message for + exceeding maximum feerate has been changed to "Fee exceeds maximum configured by user + (e.g. -maxtxfee, maxfeerate)." (#19339) + +Tests +----- + +- The BIP 325 default signet can be enabled by the `-chain=signet` or `-signet` + setting. The settings `-signetchallenge` and `-signetseednode` allow + enabling a custom signet. + +- The `generateblock` RPC allows testers using regtest mode to + generate blocks that consist of a custom set of transactions. (#17693) + +0.21.0 change log +================= + +### Consensus +- #18267 BIP-325: Signet (kallewoof) +- #20016 uint256: 1 is a constant (ajtowns) +- #20006 Fix misleading error message: Clean stack rule (sanket1729) +- #19953 Implement BIP 340-342 validation (Schnorr/taproot/tapscript) (sipa) +- #20169 Taproot follow-up: Make ComputeEntrySchnorr and ComputeEntryECDSA const to clarify contract (practicalswift) + +### Policy +- #18766 Disable fee estimation in blocksonly mode (darosior) +- #19630 Cleanup fee estimation code (darosior) +- #20165 Only relay Taproot spends if next block has it active (sipa) + +### Mining +- #17946 Fix GBT: Restore "!segwit" and "csv" to "rules" key (luke-jr) + +### Privacy +- #16432 Add privacy to the Overview page (hebasto) +- #18861 Do not answer GETDATA for to-be-announced tx (sipa) +- #18038 Mempool tracks locally submitted transactions to improve wallet privacy (amitiuttarwar) +- #19109 Only allow getdata of recently announced invs (sipa) + +### Block and transaction handling +- #17737 Add ChainstateManager, remove BlockManager global (jamesob) +- #18960 indexes: Add compact block filter headers cache (jnewbery) +- #13204 Faster sigcache nonce (JeremyRubin) +- #19088 Use std::chrono throughout some validation functions (fanquake) +- #19142 Make VerifyDB level 4 interruptible (MarcoFalke) +- #17994 Flush undo files after last block write (kallewoof) +- #18990 log: Properly log txs rejected from mempool (MarcoFalke) +- #18984 Remove unnecessary input blockfile SetPos (dgenr8) +- #19526 log: Avoid treating remote misbehvior as local system error (MarcoFalke) +- #18044 Use wtxid for transaction relay (sdaftuar) +- #18637 coins: allow cache resize after init (jamesob) +- #19854 Avoid locking CTxMemPool::cs recursively in simple cases (hebasto) +- #19478 Remove CTxMempool::mapLinks data structure member (JeremyRubin) +- #19927 Reduce direct `g_chainman` usage (dongcarl) +- #19898 log: print unexpected version warning in validation log category (n-thumann) +- #20036 signet: Add assumed values for default signet (MarcoFalke) +- #20048 chainparams: do not log signet startup messages for other chains (jonatack) +- #19339 re-delegate absurd fee checking from mempool to clients (glozow) +- #20035 signet: Fix uninitialized read in validation (MarcoFalke) +- #20157 Bugfix: chainparams: Add missing (always enabled) Taproot deployment for Signet (luke-jr) +- #20263 Update assumed chain params (MarcoFalke) +- #20372 Avoid signed integer overflow when loading a mempool.dat file with a malformed time field (practicalswift) +- #18621 script: Disallow silent bool -> cscript conversion (MarcoFalke) +- #18612, #18732 script: Remove undocumented and unused operator+ (MarcoFalke) +- #19317 Add a left-justified width field to `log2_work` component for a uniform debug.log output (jamesgmorgan) + +### P2P protocol and network code +- #18544 Limit BIP37 filter lifespan (active between `filterload`..`filterclear`) (theStack) +- #18806 Remove is{Empty,Full} flags from CBloomFilter, clarify CVE fix (theStack) +- #18512 Improve asmap checks and add sanity check (sipa) +- #18877 Serve cfcheckpt requests (jnewbery) +- #18895 Unbroadcast followups: rpcs, nLastResend, mempool sanity check (gzhao408) +- #19010 net processing: Add support for `getcfheaders` (jnewbery) +- #16939 Delay querying DNS seeds (ajtowns) +- #18807 Unbroadcast follow-ups (amitiuttarwar) +- #19044 Add support for getcfilters (jnewbery) +- #19084 improve code documentation for dns seed behaviour (ajtowns) +- #19260 disconnect peers that send filterclear + update existing filter msg disconnect logic (gzhao408) +- #19284 Add seed.bitcoin.wiz.biz to DNS seeds (wiz) +- #19322 split PushInventory() (jnewbery) +- #19204 Reduce inv traffic during IBD (MarcoFalke) +- #19470 banlist: log post-swept banlist size at startup (fanquake) +- #19191 Extract download permission from noban (MarcoFalke) +- #14033 Drop `CADDR_TIME_VERSION` checks now that `MIN_PEER_PROTO_VERSION` is greater (Empact) +- #19464 net, rpc: remove -banscore option, deprecate banscore in getpeerinfo (jonatack) +- #19514 [net/net processing] check banman pointer before dereferencing (jnewbery) +- #19512 banscore updates to gui, tests, release notes (jonatack) +- #19360 improve encapsulation of CNetAddr (vasild) +- #19217 disambiguate block-relay-only variable names from blocksonly variables (glowang) +- #19473 Add -networkactive option (hebasto) +- #19472 [net processing] Reduce `cs_main` scope in MaybeDiscourageAndDisconnect() (jnewbery) +- #19583 clean up Misbehaving() (jnewbery) +- #19534 save the network type explicitly in CNetAddr (vasild) +- #19569 Enable fetching of orphan parents from wtxid peers (sipa) +- #18991 Cache responses to GETADDR to prevent topology leaks (naumenkogs) +- #19596 Deduplicate parent txid loop of requested transactions and missing parents of orphan transactions (sdaftuar) +- #19316 Cleanup logic around connection types (amitiuttarwar) +- #19070 Signal support for compact block filters with `NODE_COMPACT_FILTERS` (jnewbery) +- #19705 Shrink CAddress from 48 to 40 bytes on x64 (vasild) +- #19704 Move ProcessMessage() to PeerLogicValidation (jnewbery) +- #19628 Change CNetAddr::ip to have flexible size (vasild) +- #19797 Remove old check for 3-byte shifted IP addresses from pre-0.2.9 nodes (#19797) +- #19607 Add Peer struct for per-peer data in net processing (jnewbery) +- #19857 improve nLastBlockTime and nLastTXTime documentation (jonatack) +- #19724 Cleanup connection types- followups (amitiuttarwar) +- #19670 Protect localhost and block-relay-only peers from eviction (sdaftuar) +- #19728 Increase the ip address relay branching factor for unreachable networks (sipa) +- #19879 Miscellaneous wtxid followups (amitiuttarwar) +- #19697 Improvements on ADDR caching (naumenkogs) +- #17785 Unify Send and Receive protocol versions (hebasto) +- #19845 CNetAddr: add support to (un)serialize as ADDRv2 (vasild) +- #19107 Move all header verification into the network layer, extend logging (troygiorshev) +- #20003 Exit with error message if -proxy is specified without arguments (instead of continuing without proxy server) (practicalswift) +- #19991 Use alternative port for incoming Tor connections (hebasto) +- #19723 Ignore unknown messages before VERACK (sdaftuar) +- #19954 Complete the BIP155 implementation and upgrade to TORv3 (vasild) +- #20119 BIP155 follow-ups (sipa) +- #19988 Overhaul transaction request logic (sipa) +- #17428 Try to preserve outbound block-relay-only connections during restart (hebasto) +- #19911 Guard `vRecvGetData` with `cs_vRecv` and `orphan_work_set` with `g_cs_orphans` (narula) +- #19753 Don't add AlreadyHave transactions to recentRejects (troygiorshev) +- #20187 Test-before-evict bugfix and improvements for block-relay-only peers (sdaftuar) +- #20237 Hardcoded seeds update for 0.21 (laanwj) +- #20212 Fix output of peer address in version message (vasild) +- #20284 Ensure old versions don't parse peers.dat (vasild) +- #20405 Avoid calculating onion address checksum when version is not 3 (lontivero) +- #20564 Don't send 'sendaddrv2' to pre-70016 software, and send before 'verack' (sipa) +- #20660 Move signet onion seed from v2 to v3 (Sjors) + +### Wallet +- #18262 Exit selection when `best_waste` is 0 (achow101) +- #17824 Prefer full destination groups in coin selection (fjahr) +- #17219 Allow transaction without change if keypool is empty (Sjors) +- #15761 Replace -upgradewallet startup option with upgradewallet RPC (achow101) +- #18671 Add BlockUntilSyncedToCurrentChain to dumpwallet (MarcoFalke) +- #16528 Native Descriptor Wallets using DescriptorScriptPubKeyMan (achow101) +- #18777 Recommend absolute path for dumpwallet (MarcoFalke) +- #16426 Reverse `cs_main`, `cs_wallet` lock order and reduce `cs_main` locking (ariard) +- #18699 Avoid translating RPC errors (MarcoFalke) +- #18782 Make sure no DescriptorScriptPubKeyMan or WalletDescriptor members are left uninitialized after construction (practicalswift) +- #9381 Remove CWalletTx merging logic from AddToWallet (ryanofsky) +- #16946 Include a checksum of encrypted private keys (achow101) +- #17681 Keep inactive seeds after sethdseed and derive keys from them as needed (achow101) +- #18918 Move salvagewallet into wallettool (achow101) +- #14988 Fix for confirmed column in csv export for payment to self transactions (benthecarman) +- #18275 Error if an explicit fee rate was given but the needed fee rate differed (kallewoof) +- #19054 Skip hdKeypath of 'm' when determining inactive hd seeds (achow101) +- #17938 Disallow automatic conversion between disparate hash types (Empact) +- #19237 Check size after unserializing a pubkey (elichai) +- #11413 sendtoaddress/sendmany: Add explicit feerate option (kallewoof) +- #18850 Fix ZapSelectTx to sync wallet spends (bvbfan) +- #18923 Never schedule MaybeCompactWalletDB when `-flushwallet` is off (MarcoFalke) +- #19441 walletdb: Don't reinitialize desc cache with multiple cache entries (achow101) +- #18907 walletdb: Don't remove database transaction logs and instead error (achow101) +- #19334 Introduce WalletDatabase abstract class (achow101) +- #19335 Cleanup and separate BerkeleyDatabase and BerkeleyBatch (achow101) +- #19102 Introduce and use DummyDatabase instead of dummy BerkeleyDatabase (achow101) +- #19568 Wallet should not override signing errors (fjahr) +- #17204 Do not turn `OP_1NEGATE` in scriptSig into `0x0181` in signing code (sipa) (meshcollider) +- #19457 Cleanup wallettool salvage and walletdb extraneous declarations (achow101) +- #15937 Add loadwallet and createwallet `load_on_startup` options (ryanofsky) +- #16841 Replace GetScriptForWitness with GetScriptForDestination (meshcollider) +- #14582 always do avoid partial spends if fees are within a specified range (kallewoof) +- #19743 -maxapsfee follow-up (kallewoof) +- #19289 GetWalletTx and IsMine require `cs_wallet` lock (promag) +- #19671 Remove -zapwallettxes (achow101) +- #19805 Avoid deserializing unused records when salvaging (achow101) +- #19754 wallet, gui: Reload previously loaded wallets on startup (achow101) +- #19738 Avoid multiple BerkeleyBatch in DelAddressBook (promag) +- #19919 bugfix: make LoadWallet assigns status always (AkioNak) +- #16378 The ultimate send RPC (Sjors) +- #15454 Remove the automatic creation and loading of the default wallet (achow101) +- #19501 `send*` RPCs in the wallet returns the "fee reason" (stackman27) +- #20130 Remove db mode string (S3RK) +- #19077 Add sqlite as an alternative wallet database and use it for new descriptor wallets (achow101) +- #20125 Expose database format in getwalletinfo (promag) +- #20198 Show name, format and if uses descriptors in bitcoin-wallet tool (jonasschnelli) +- #20216 Fix buffer over-read in SQLite file magic check (theStack) +- #20186 Make -wallet setting not create wallets (ryanofsky) +- #20230 Fix bug when just created encrypted wallet cannot get address (hebasto) +- #20282 Change `upgradewallet` return type to be an object (jnewbery) +- #20220 Explicit fee rate follow-ups/fixes for 0.21 (jonatack) +- #20199 Ignore (but warn) on duplicate -wallet parameters (jonasschnelli) +- #20324 Set DatabaseStatus::SUCCESS in MakeSQLiteDatabase (MarcoFalke) +- #20266 Fix change detection of imported internal descriptors (achow101) +- #20153 Do not import a descriptor with hardened derivations into a watch-only wallet (S3RK) +- #20344 Fix scanning progress calculation for single block range (theStack) +- #19502 Bugfix: Wallet: Soft-fail exceptions within ListWalletDir file checks (luke-jr) +- #20378 Fix potential division by 0 in WalletLogPrintf (jonasschnelli) +- #18836 Upgradewallet fixes and additional tests (achow101) +- #20139 Do not return warnings from UpgradeWallet() (stackman27) +- #20305 Introduce `fee_rate` sat/vB param/option (jonatack) +- #20426 Allow zero-fee fundrawtransaction/walletcreatefundedpsbt and other fixes (jonatack) +- #20573 wallet, bugfix: allow send with string `fee_rate` amounts (jonatack) + +### RPC and other APIs +- #18574 cli: Call getbalances.ismine.trusted instead of getwalletinfo.balance (jonatack) +- #17693 Add `generateblock` to mine a custom set of transactions (andrewtoth) +- #18495 Remove deprecated migration code (vasild) +- #18493 Remove deprecated "size" from mempool txs (vasild) +- #18467 Improve documentation and return value of settxfee (fjahr) +- #18607 Fix named arguments in documentation (MarcoFalke) +- #17831 doc: Fix and extend getblockstats examples (asoltys) +- #18785 Prevent valgrind false positive in `rest_blockhash_by_height` (ryanofsky) +- #18999 log: Remove "No rpcpassword set" from logs (MarcoFalke) +- #19006 Avoid crash when `g_thread_http` was never started (MarcoFalke) +- #18594 cli: Display multiwallet balances in -getinfo (jonatack) +- #19056 Make gettxoutsetinfo/GetUTXOStats interruptible (MarcoFalke) +- #19112 Remove special case for unknown service flags (MarcoFalke) +- #18826 Expose txinwitness for coinbase in JSON form from RPC (rvagg) +- #19282 Rephrase generatetoaddress help, and use `PACKAGE_NAME` (luke-jr) +- #16377 don't automatically append inputs in walletcreatefundedpsbt (Sjors) +- #19200 Remove deprecated getaddressinfo fields (jonatack) +- #19133 rpc, cli, test: add bitcoin-cli -generate command (jonatack) +- #19469 Deprecate banscore field in getpeerinfo (jonatack) +- #16525 Dump transaction version as an unsigned integer in RPC/TxToUniv (TheBlueMatt) +- #19555 Deduplicate WriteHDKeypath() used in decodepsbt (theStack) +- #19589 Avoid useless mempool query in gettxoutproof (MarcoFalke) +- #19585 RPCResult Type of MempoolEntryDescription should be OBJ (stylesuxx) +- #19634 Document getwalletinfo's `unlocked_until` field as optional (justinmoon) +- #19658 Allow RPC to fetch all addrman records and add records to addrman (jnewbery) +- #19696 Fix addnode remove command error (fjahr) +- #18654 Separate bumpfee's psbt creation function into psbtbumpfee (achow101) +- #19655 Catch listsinceblock `target_confirmations` exceeding block count (adaminsky) +- #19644 Document returned error fields as optional if applicable (theStack) +- #19455 rpc generate: print useful help and error message (jonatack) +- #19550 Add listindices RPC (fjahr) +- #19169 Validate provided keys for `query_options` parameter in listunspent (PastaPastaPasta) +- #18244 fundrawtransaction and walletcreatefundedpsbt also lock manually selected coins (Sjors) +- #14687 zmq: Enable TCP keepalive (mruddy) +- #19405 Add network in/out connections to `getnetworkinfo` and `-getinfo` (jonatack) +- #19878 rawtransaction: Fix argument in combinerawtransaction help message (pinheadmz) +- #19940 Return fee and vsize from testmempoolaccept (gzhao408) +- #13686 zmq: Small cleanups in the ZMQ code (domob1812) +- #19386, #19528, #19717, #19849, #19994 Assert that RPCArg names are equal to CRPCCommand ones (MarcoFalke) +- #19725 Add connection type to getpeerinfo, improve logs (amitiuttarwar) +- #19969 Send RPC bug fix and touch-ups (Sjors) +- #18309 zmq: Add support to listen on multiple interfaces (n-thumann) +- #20055 Set HTTP Content-Type in bitcoin-cli (laanwj) +- #19956 Improve invalid vout value rpc error message (n1rna) +- #20101 Change no wallet loaded message to be clearer (achow101) +- #19998 Add `via_tor` to `getpeerinfo` output (hebasto) +- #19770 getpeerinfo: Deprecate "whitelisted" field (replaced by "permissions") (luke-jr) +- #20120 net, rpc, test, bugfix: update GetNetworkName, GetNetworksInfo, regression tests (jonatack) +- #20595 Improve heuristic hex transaction decoding (sipa) +- #20731 Add missing description of vout in getrawtransaction help text (benthecarman) +- #19328 Add gettxoutsetinfo `hash_type` option (fjahr) +- #19731 Expose nLastBlockTime/nLastTXTime as last `block/last_transaction` in getpeerinfo (jonatack) +- #19572 zmq: Create "sequence" notifier, enabling client-side mempool tracking (instagibbs) +- #20002 Expose peer network in getpeerinfo; simplify/improve -netinfo (jonatack) + +### GUI +- #17905 Avoid redundant tx status updates (ryanofsky) +- #18646 Use `PACKAGE_NAME` in exception message (fanquake) +- #17509 Save and load PSBT (Sjors) +- #18769 Remove bug fix for Qt < 5.5 (10xcryptodev) +- #15768 Add close window shortcut (IPGlider) +- #16224 Bilingual GUI error messages (hebasto) +- #18922 Do not translate InitWarning messages in debug.log (hebasto) +- #18152 Use NotificationStatus enum for signals to GUI (hebasto) +- #18587 Avoid wallet tryGetBalances calls in WalletModel::pollBalanceChanged (ryanofsky) +- #17597 Fix height of QR-less ReceiveRequestDialog (hebasto) +- #17918 Hide non PKHash-Addresses in signing address book (emilengler) +- #17956 Disable unavailable context menu items in transactions tab (kristapsk) +- #17968 Ensure that ModalOverlay is resized properly (hebasto) +- #17993 Balance/TxStatus polling update based on last block hash (furszy) +- #18424 Use parent-child relation to manage lifetime of OptionsModel object (hebasto) +- #18452 Fix shutdown when `waitfor*` cmds are called from RPC console (hebasto) +- #15202 Add Close All Wallets action (promag) +- #19132 lock `cs_main`, `m_cached_tip_mutex` in that order (vasild) +- #18898 Display warnings as rich text (hebasto) +- #19231 add missing translation.h include to fix build (fanquake) +- #18027 "PSBT Operations" dialog (gwillen) +- #19256 Change combiner for signals to `optional_last_value` (fanquake) +- #18896 Reset toolbar after all wallets are closed (hebasto) +- #18993 increase console command max length (10xcryptodev) +- #19323 Fix regression in *txoutset* in GUI console (hebasto) +- #19210 Get rid of cursor in out-of-focus labels (hebasto) +- #19011 Reduce `cs_main` lock accumulation during GUI startup (jonasschnelli) +- #19844 Remove usage of boost::bind (fanquake) +- #20479 Fix QPainter non-determinism on macOS (0.21 backport) (laanwj) +- gui#6 Do not truncate node flag strings in debugwindow peers details tab (Saibato) +- gui#8 Fix regression in TransactionTableModel (hebasto) +- gui#17 doc: Remove outdated comment in TransactionTablePriv (MarcoFalke) +- gui#20 Wrap tooltips in the intro window (hebasto) +- gui#30 Disable the main window toolbar when the modal overlay is shown (hebasto) +- gui#34 Show permissions instead of whitelisted (laanwj) +- gui#35 Parse params directly instead of through node (ryanofsky) +- gui#39 Add visual accenting for the 'Create new receiving address' button (hebasto) +- gui#40 Clarify block height label (hebasto) +- gui#43 bugfix: Call setWalletActionsEnabled(true) only for the first wallet (hebasto) +- gui#97 Relax GUI freezes during IBD (jonasschnelli) +- gui#71 Fix visual quality of text in QR image (hebasto) +- gui#96 Slight improve create wallet dialog (Sjors) +- gui#102 Fix SplashScreen crash when run with -disablewallet (hebasto) +- gui#116 Fix unreasonable default size of the main window without loaded wallets (hebasto) +- gui#120 Fix multiwallet transaction notifications (promag) + +### Build system +- #18504 Drop bitcoin-tx and bitcoin-wallet dependencies on libevent (ryanofsky) +- #18586 Bump gitian descriptors to 0.21 (laanwj) +- #17595 guix: Enable building for `x86_64-w64-mingw32` target (dongcarl) +- #17929 add linker optimisation flags to gitian & guix (Linux) (fanquake) +- #18556 Drop make dist in gitian builds (hebasto) +- #18088 ensure we aren't using GNU extensions (fanquake) +- #18741 guix: Make source tarball using git-archive (dongcarl) +- #18843 warn on potentially uninitialized reads (vasild) +- #17874 make linker checks more robust (fanquake) +- #18535 remove -Qunused-arguments workaround for clang + ccache (fanquake) +- #18743 Add --sysroot option to mac os native compile flags (ryanofsky) +- #18216 test, build: Enable -Werror=sign-compare (Empact) +- #18928 don't pass -w when building for Windows (fanquake) +- #16710 Enable -Wsuggest-override if available (hebasto) +- #18738 Suppress -Wdeprecated-copy warnings (hebasto) +- #18862 Remove fdelt_chk back-compat code and sanity check (fanquake) +- #18887 enable -Werror=gnu (vasild) +- #18956 enforce minimum required Windows version (7) (fanquake) +- #18958 guix: Make V=1 more powerful for debugging (dongcarl) +- #18677 Multiprocess build support (ryanofsky) +- #19094 Only allow ASCII identifiers (laanwj) +- #18820 Propagate well-known vars into depends (dongcarl) +- #19173 turn on --enable-c++17 by --enable-fuzz (vasild) +- #18297 Use pkg-config in BITCOIN_QT_CONFIGURE for all hosts including Windows (hebasto) +- #19301 don't warn when doxygen isn't found (fanquake) +- #19240 macOS toolchain simplification and bump (dongcarl) +- #19356 Fix search for brew-installed BDB 4 on OS X (gwillen) +- #19394 Remove unused `RES_IMAGES` (Bushstar) +- #19403 improve `__builtin_clz*` detection (fanquake) +- #19375 target Windows 7 when building libevent and fix ipv6 usage (fanquake) +- #19331 Do not include server symbols in wallet (MarcoFalke) +- #19257 remove BIP70 configure option (fanquake) +- #18288 Add MemorySanitizer (MSan) in Travis to detect use of uninitialized memory (practicalswift) +- #18307 Require pkg-config for all of the hosts (hebasto) +- #19445 Update msvc build to use ISO standard C++17 (sipsorcery) +- #18882 fix -Wformat-security check when compiling with GCC (fanquake) +- #17919 Allow building with system clang (dongcarl) +- #19553 pass -fcommon when building genisoimage (fanquake) +- #19565 call `AC_PATH_TOOL` for dsymutil in macOS cross-compile (fanquake) +- #19530 build LTO support into Apple's ld64 (theuni) +- #19525 add -Wl,-z,separate-code to hardening flags (fanquake) +- #19667 set minimum required Boost to 1.58.0 (fanquake) +- #19672 make clean removes .gcda and .gcno files from fuzz directory (Crypt-iQ) +- #19622 Drop ancient hack in gitian-linux descriptor (hebasto) +- #19688 Add support for llvm-cov (hebasto) +- #19718 Add missed gcov files to 'make clean' (hebasto) +- #19719 Add Werror=range-loop-analysis (MarcoFalke) +- #19015 Enable some commonly enabled compiler diagnostics (practicalswift) +- #19689 build, qt: Add Qt version checking (hebasto) +- #17396 modest Android improvements (icota) +- #18405 Drop all of the ZeroMQ patches (hebasto) +- #15704 Move Win32 defines to configure.ac to ensure they are globally defined (luke-jr) +- #19761 improve sed robustness by not using sed (fanquake) +- #19758 Drop deprecated and unused `GUARDED_VAR` and `PT_GUARDED_VAR` annotations (hebasto) +- #18921 add stack-clash and control-flow protection options to hardening flags (fanquake) +- #19803 Bugfix: Define and use `HAVE_FDATASYNC` correctly outside LevelDB (luke-jr) +- #19685 CMake invocation cleanup (dongcarl) +- #19861 add /usr/local/ to `LCOV_FILTER_PATTERN` for macOS builds (Crypt-iQ) +- #19916 allow user to specify `DIR_FUZZ_SEED_CORPUS` for `cov_fuzz` (Crypt-iQ) +- #19944 Update secp256k1 subtree (including BIP340 support) (sipa) +- #19558 Split pthread flags out of ldflags and dont use when building libconsensus (fanquake) +- #19959 patch qt libpng to fix powerpc build (fanquake) +- #19868 Fix target name (hebasto) +- #19960 The vcpkg tool has introduced a proper way to use manifests (sipsorcery) +- #20065 fuzz: Configure check for main function (MarcoFalke) +- #18750 Optionally skip external warnings (vasild) +- #20147 Update libsecp256k1 (endomorphism, test improvements) (sipa) +- #20156 Make sqlite support optional (compile-time) (luke-jr) +- #20318 Ensure source tarball has leading directory name (MarcoFalke) +- #20447 Patch `qt_intersect_spans` to avoid non-deterministic behavior in LLVM 8 (achow101) +- #20505 Avoid secp256k1.h include from system (dergoegge) +- #20527 Do not ignore Homebrew's SQLite on macOS (hebasto) +- #20478 Don't set BDB flags when configuring without (jonasschnelli) +- #20563 Check that Homebrew's berkeley-db4 package is actually installed (hebasto) +- #19493 Fix clang build on Mac (bvbfan) + +### Tests and QA +- #18593 Complete impl. of `msg_merkleblock` and `wait_for_merkleblock` (theStack) +- #18609 Remove REJECT message code (hebasto) +- #18584 Check that the version message does not leak the local address (MarcoFalke) +- #18597 Extend `wallet_dump` test to cover comments (MarcoFalke) +- #18596 Try once more when RPC connection fails on Windows (MarcoFalke) +- #18451 shift coverage from getunconfirmedbalance to getbalances (jonatack) +- #18631 appveyor: Disable functional tests for now (MarcoFalke) +- #18628 Add various low-level p2p tests (MarcoFalke) +- #18615 Avoid accessing free'd memory in `validation_chainstatemanager_tests` (MarcoFalke) +- #18571 fuzz: Disable debug log file (MarcoFalke) +- #18653 add coverage for bitcoin-cli -rpcwait (jonatack) +- #18660 Verify findCommonAncestor always initializes outputs (ryanofsky) +- #17669 Have coins simulation test also use CCoinsViewDB (jamesob) +- #18662 Replace gArgs with local argsman in bench (MarcoFalke) +- #18641 Create cached blocks not in the future (MarcoFalke) +- #18682 fuzz: `http_request` workaround for libevent < 2.1.1 (theStack) +- #18692 Bump timeout in `wallet_import_rescan` (MarcoFalke) +- #18695 Replace boost::mutex with std::mutex (hebasto) +- #18633 Properly raise FailedToStartError when rpc shutdown before warmup finished (MarcoFalke) +- #18675 Don't initialize PrecomputedTransactionData in txvalidationcache tests (jnewbery) +- #18691 Add `wait_for_cookie_credentials()` to framework for rpcwait tests (jonatack) +- #18672 Add further BIP37 size limit checks to `p2p_filter.py` (theStack) +- #18721 Fix linter issue (hebasto) +- #18384 More specific `feature_segwit` test error messages and fixing incorrect comments (gzhao408) +- #18575 bench: Remove requirement that all benches use same testing setup (MarcoFalke) +- #18690 Check object hashes in `wait_for_getdata` (robot-visions) +- #18712 display command line options passed to `send_cli()` in debug log (jonatack) +- #18745 Check submitblock return values (MarcoFalke) +- #18756 Use `wait_for_getdata()` in `p2p_compactblocks.py` (theStack) +- #18724 Add coverage for -rpcwallet cli option (jonatack) +- #18754 bench: Add caddrman benchmarks (vasild) +- #18585 Use zero-argument super() shortcut (Python 3.0+) (theStack) +- #18688 fuzz: Run in parallel (MarcoFalke) +- #18770 Remove raw-tx byte juggling in `mempool_reorg` (MarcoFalke) +- #18805 Add missing `sync_all` to `wallet_importdescriptors.py` (achow101) +- #18759 bench: Start nodes with -nodebuglogfile (MarcoFalke) +- #18774 Added test for upgradewallet RPC (brakmic) +- #18485 Add `mempool_updatefromblock.py` (hebasto) +- #18727 Add CreateWalletFromFile test (ryanofsky) +- #18726 Check misbehavior more independently in `p2p_filter.py` (robot-visions) +- #18825 Fix message for `ECC_InitSanityCheck` test (fanquake) +- #18576 Use unittest for `test_framework` unit testing (gzhao408) +- #18828 Strip down previous releases boilerplate (MarcoFalke) +- #18617 Add factor option to adjust test timeouts (brakmic) +- #18855 `feature_backwards_compatibility.py` test downgrade after upgrade (achow101) +- #18864 Add v0.16.3 backwards compatibility test, bump v0.19.0.1 to v0.19.1 (Sjors) +- #18917 fuzz: Fix vector size problem in system fuzzer (brakmic) +- #18901 fuzz: use std::optional for `sep_pos_opt` variable (brakmic) +- #18888 Remove RPCOverloadWrapper boilerplate (MarcoFalke) +- #18952 Avoid os-dependent path (fametrano) +- #18938 Fill fuzzing coverage gaps for functions in consensus/validation.h, primitives/block.h and util/translation.h (practicalswift) +- #18986 Add capability to disable RPC timeout in functional tests (rajarshimaitra) +- #18530 Add test for -blocksonly and -whitelistforcerelay param interaction (glowang) +- #19014 Replace `TEST_PREVIOUS_RELEASES` env var with `test_framework` option (MarcoFalke) +- #19052 Don't limit fuzzing inputs to 1 MB for afl-fuzz (now: ∞ ∀ fuzzers) (practicalswift) +- #19060 Remove global `wait_until` from `p2p_getdata` (MarcoFalke) +- #18926 Pass ArgsManager into `getarg_tests` (glowang) +- #19110 Explain that a bug should be filed when the tests fail (MarcoFalke) +- #18965 Implement `base58_decode` (10xcryptodev) +- #16564 Always define the `raii_event_tests` test suite (candrews) +- #19122 Add missing `sync_blocks` to `wallet_hd` (MarcoFalke) +- #18875 fuzz: Stop nodes in `process_message*` fuzzers (MarcoFalke) +- #18974 Check that invalid witness destinations can not be imported (MarcoFalke) +- #18210 Type hints in Python tests (kiminuo) +- #19159 Make valgrind.supp work on aarch64 (MarcoFalke) +- #19082 Moved the CScriptNum asserts into the unit test in script.py (gillichu) +- #19172 Do not swallow flake8 exit code (hebasto) +- #19188 Avoid overwriting the NodeContext member of the testing setup [-Wshadow-field] (MarcoFalke) +- #18890 `disconnect_nodes` should warn if nodes were already disconnected (robot-visions) +- #19227 change blacklist to blocklist (TrentZ) +- #19230 Move base58 to own module to break circular dependency (sipa) +- #19083 `msg_mempool`, `fRelay`, and other bloomfilter tests (gzhao408) +- #16756 Connection eviction logic tests (mzumsande) +- #19177 Fix and clean `p2p_invalid_messages` functional tests (troygiorshev) +- #19264 Don't import asyncio to test magic bytes (jnewbery) +- #19178 Make `mininode_lock` non-reentrant (jnewbery) +- #19153 Mempool compatibility test (S3RK) +- #18434 Add a test-security target and run it in CI (fanquake) +- #19252 Wait for disconnect in `disconnect_p2ps` + bloomfilter test followups (gzhao408) +- #19298 Add missing `sync_blocks` (MarcoFalke) +- #19304 Check that message sends successfully when header is split across two buffers (troygiorshev) +- #19208 move `sync_blocks` and `sync_mempool` functions to `test_framework.py` (ycshao) +- #19198 Check that peers with forcerelay permission are not asked to feefilter (MarcoFalke) +- #19351 add two edge case tests for CSubNet (vasild) +- #19272 net, test: invalid p2p messages and test framework improvements (jonatack) +- #19348 Bump linter versions (duncandean) +- #19366 Provide main(…) function in fuzzer. Allow building uninstrumented harnesses with --enable-fuzz (practicalswift) +- #19412 move `TEST_RUNNER_EXTRA` into native tsan setup (fanquake) +- #19368 Improve functional tests compatibility with BSD/macOS (S3RK) +- #19028 Set -logthreadnames in unit tests (MarcoFalke) +- #18649 Add std::locale::global to list of locale dependent functions (practicalswift) +- #19140 Avoid fuzzer-specific nullptr dereference in libevent when handling PROXY requests (practicalswift) +- #19214 Auto-detect SHA256 implementation in benchmarks (sipa) +- #19353 Fix mistakenly swapped "previous" and "current" lock orders (hebasto) +- #19533 Remove unnecessary `cs_mains` in `denialofservice_tests` (jnewbery) +- #19423 add functional test for txrelay during and after IBD (gzhao408) +- #16878 Fix non-deterministic coverage of test `DoS_mapOrphans` (davereikher) +- #19548 fuzz: add missing overrides to `signature_checker` (jonatack) +- #19562 Fix fuzzer compilation on macOS (freenancial) +- #19370 Static asserts for consistency of fee defaults (domob1812) +- #19599 clean `message_count` and `last_message` (troygiorshev) +- #19597 test decodepsbt fee calculation (count input value only once per UTXO) (theStack) +- #18011 Replace current benchmarking framework with nanobench (martinus) +- #19489 Fail `wait_until` early if connection is lost (MarcoFalke) +- #19340 Preserve the `LockData` initial state if "potential deadlock detected" exception thrown (hebasto) +- #19632 Catch decimal.InvalidOperation from `TestNodeCLI#send_cli` (Empact) +- #19098 Remove duplicate NodeContext hacks (ryanofsky) +- #19649 Restore test case for p2p transaction blinding (instagibbs) +- #19657 Wait until `is_connected` in `add_p2p_connection` (MarcoFalke) +- #19631 Wait for 'cmpctblock' in `p2p_compactblocks` when it is expected (Empact) +- #19674 use throwaway _ variable for unused loop counters (theStack) +- #19709 Fix 'make cov' with clang (hebasto) +- #19564 `p2p_feefilter` improvements (logging, refactoring, speedup) (theStack) +- #19756 add `sync_all` to fix race condition in wallet groups test (kallewoof) +- #19727 Removing unused classes from `p2p_leak.py` (dhruv) +- #19722 Add test for getblockheader verboseness (torhte) +- #19659 Add a seed corpus generation option to the fuzzing `test_runner` (darosior) +- #19775 Activate segwit in TestChain100Setup (MarcoFalke) +- #19760 Remove confusing mininode terminology (jnewbery) +- #19752 Update `wait_until` usage in tests not to use the one from utils (slmtpz) +- #19839 Set appveyor VM version to previous Visual Studio 2019 release (sipsorcery) +- #19830 Add tsan supp for leveldb::DBImpl::DeleteObsoleteFiles (MarcoFalke) +- #19710 bench: Prevent thread oversubscription and decreases the variance of result values (hebasto) +- #19842 Update the vcpkg checkout commit ID in appveyor config (sipsorcery) +- #19507 Expand functional zmq transaction tests (instagibbs) +- #19816 Rename wait until helper to `wait_until_helper` (MarcoFalke) +- #19859 Fixes failing functional test by changing version (n-thumann) +- #19887 Fix flaky `wallet_basic` test (fjahr) +- #19897 Change `FILE_CHAR_BLOCKLIST` to `FILE_CHARS_DISALLOWED` (verretor) +- #19800 Mockwallet (MarcoFalke) +- #19922 Run `rpc_txoutproof.py` even with wallet disabled (MarcoFalke) +- #19936 batch rpc with params (instagibbs) +- #19971 create default wallet in extended tests (Sjors) +- #19781 add parameterized constructor for `msg_sendcmpct()` (theStack) +- #19963 Clarify blocksonly whitelistforcerelay test (t-bast) +- #20022 Use explicit p2p objects where available (guggero) +- #20028 Check that invalid peer traffic is accounted for (MarcoFalke) +- #20004 Add signet witness commitment section parse tests (MarcoFalke) +- #20034 Get rid of default wallet hacks (ryanofsky) +- #20069 Mention commit id in scripted diff error (laanwj) +- #19947 Cover `change_type` option of "walletcreatefundedpsbt" RPC (guggero) +- #20126 `p2p_leak_tx.py` improvements (use MiniWallet, add `p2p_lock` acquires) (theStack) +- #20129 Don't export `in6addr_loopback` (vasild) +- #20131 Remove unused nVersion=1 in p2p tests (MarcoFalke) +- #20161 Minor Taproot follow-ups (sipa) +- #19401 Use GBT to get block versions correct (luke-jr) +- #20159 `mining_getblocktemplate_longpoll.py` improvements (use MiniWallet, add logging) (theStack) +- #20039 Convert amounts from float to decimal (prayank23) +- #20112 Speed up `wallet_resendwallettransactions` with mockscheduler RPC (MarcoFalke) +- #20247 fuzz: Check for addrv1 compatibility before using addrv1 serializer. Fuzz addrv2 serialization (practicalswift) +- #20167 Add test for -blockversion (MarcoFalke) +- #19877 Clarify `rpc_net` & `p2p_disconnect_ban functional` tests (amitiuttarwar) +- #20258 Remove getnettotals/getpeerinfo consistency test (jnewbery) +- #20242 fuzz: Properly initialize PrecomputedTransactionData (MarcoFalke) +- #20262 Skip --descriptor tests if sqlite is not compiled (achow101) +- #18788 Update more tests to work with descriptor wallets (achow101) +- #20289 fuzz: Check for addrv1 compatibility before using addrv1 serializer/deserializer on CService (practicalswift) +- #20290 fuzz: Fix DecodeHexTx fuzzing harness issue (practicalswift) +- #20245 Run `script_assets_test` even if built --with-libs=no (MarcoFalke) +- #20300 fuzz: Add missing `ECC_Start` to `descriptor_parse` test (S3RK) +- #20283 Only try witness deser when checking for witness deser failure (MarcoFalke) +- #20303 fuzz: Assert expected DecodeHexTx behaviour when using legacy decoding (practicalswift) +- #20316 Fix `wallet_multiwallet` test issue on Windows (MarcoFalke) +- #20326 Fix `ecdsa_verify` in test framework (stepansnigirev) +- #20328 cirrus: Skip tasks on the gui repo main branch (MarcoFalke) +- #20355 fuzz: Check for addrv1 compatibility before using addrv1 serializer/deserializer on CSubNet (practicalswift) +- #20332 Mock IBD in `net_processing` fuzzers (MarcoFalke) +- #20218 Suppress `epoll_ctl` data race (MarcoFalke) +- #20375 fuzz: Improve coverage for CPartialMerkleTree fuzzing harness (practicalswift) +- #19669 contrib: Fixup valgrind suppressions file (MarcoFalke) +- #18879 valgrind: remove outdated suppressions (fanquake) +- #19226 Add BerkeleyDatabase tsan suppression (MarcoFalke) +- #20379 Remove no longer needed UBSan suppression (float divide-by-zero in validation.cpp) (practicalswift) +- #18190, #18736, #18744, #18775, #18783, #18867, #18994, #19065, + #19067, #19143, #19222, #19247, #19286, #19296, #19379, #19934, + #20188, #20395 Add fuzzing harnessses (practicalswift) +- #18638 Use mockable time for ping/pong, add tests (MarcoFalke) +- #19951 CNetAddr scoped ipv6 test coverage, rename scopeId to `m_scope_id` (jonatack) +- #20027 Use mockable time everywhere in `net_processing` (sipa) +- #19105 Add Muhash3072 implementation in Python (fjahr) +- #18704, #18752, #18753, #18765, #18839, #18866, #18873, #19022, + #19023, #19429, #19552, #19778, #20176, #20179, #20214, #20292, + #20299, #20322 Fix intermittent test issues (MarcoFalke) +- #20390 CI/Cirrus: Skip `merge_base` step for non-PRs (luke-jr) +- #18634 ci: Add fuzzbuzz integration configuration file (practicalswift) +- #18591 Add C++17 build to Travis (sipa) +- #18581, #18667, #18798, #19495, #19519, #19538 CI improvements (hebasto) +- #18683, #18705, #18735, #18778, #18799, #18829, #18912, #18929, + #19008, #19041, #19164, #19201, #19267, #19276, #19321, #19371, + #19427, #19730, #19746, #19881, #20294, #20339, #20368 CI improvements (MarcoFalke) +- #20489, #20506 MSVC CI improvements (sipsorcery) + +### Miscellaneous +- #18713 scripts: Add macho stack canary check to security-check.py (fanquake) +- #18629 scripts: Add pe .reloc section check to security-check.py (fanquake) +- #18437 util: `Detect posix_fallocate()` instead of assuming (vasild) +- #18413 script: Prevent ub when computing abs value for num opcode serialize (pierreN) +- #18443 lockedpool: avoid sensitive data in core files (FreeBSD) (vasild) +- #18885 contrib: Move optimize-pngs.py script to the maintainer repo (MarcoFalke) +- #18317 Serialization improvements step 6 (all except wallet/gui) (sipa) +- #16127 More thread safety annotation coverage (ajtowns) +- #19228 Update libsecp256k1 subtree (sipa) +- #19277 util: Add assert identity function (MarcoFalke) +- #19491 util: Make assert work with any value (MarcoFalke) +- #19205 script: `previous_release.sh` rewritten in python (bliotti) +- #15935 Add <datadir>/settings.json persistent settings storage (ryanofsky) +- #19439 script: Linter to check commit message formatting (Ghorbanian) +- #19654 lint: Improve commit message linter in travis (fjahr) +- #15382 util: Add runcommandparsejson (Sjors) +- #19614 util: Use `have_fdatasync` to determine fdatasync() use (fanquake) +- #19813 util, ci: Hard code previous release tarball checksums (hebasto) +- #19841 Implement Keccak and `SHA3_256` (sipa) +- #19643 Add -netinfo peer connections dashboard (jonatack) +- #15367 feature: Added ability for users to add a startup command (benthecarman) +- #19984 log: Remove static log message "Initializing chainstate Chainstate [ibd] @ height -1 (null)" (practicalswift) +- #20092 util: Do not use gargs global in argsmanager member functions (hebasto) +- #20168 contrib: Fix `gen_key_io_test_vectors.py` imports (MarcoFalke) +- #19624 Warn on unknown `rw_settings` (MarcoFalke) +- #20257 Update secp256k1 subtree to latest master (sipa) +- #20346 script: Modify security-check.py to use "==" instead of "is" for literal comparison (tylerchambers) +- #18881 Prevent UB in DeleteLock() function (hebasto) +- #19180, #19189, #19190, #19220, #19399 Replace RecursiveMutex with Mutex (hebasto) +- #19347 Make `cs_inventory` nonrecursive (jnewbery) +- #19773 Avoid recursive lock in IsTrusted (promag) +- #18790 Improve thread naming (hebasto) +- #20140 Restore compatibility with old CSubNet serialization (sipa) +- #17775 DecodeHexTx: Try case where txn has inputs first (instagibbs) + +### Documentation +- #18502 Update docs for getbalance (default minconf should be 0) (uzyn) +- #18632 Fix macos comments in release-notes (MarcoFalke) +- #18645 Update thread information in developer docs (jnewbery) +- #18709 Note why we can't use `thread_local` with glibc back compat (fanquake) +- #18410 Improve commenting for coins.cpp|h (jnewbery) +- #18157 fixing init.md documentation to not require rpcpassword (jkcd) +- #18739 Document how to fuzz Bitcoin Core using Honggfuzz (practicalswift) +- #18779 Better explain GNU ld's dislike of ld64's options (fanquake) +- #18663 Mention build docs in README.md (saahilshangle) +- #18810 Update rest info on block size and json (chrisabrams) +- #18939 Add c++17-enable flag to fuzzing instructions (mzumsande) +- #18957 Add a link from ZMQ doc to ZMQ example in contrib/ (meeDamian) +- #19058 Drop protobuf stuff (hebasto) +- #19061 Add link to Visual Studio build readme (maitrebitcoin) +- #19072 Expand section on Getting Started (MarcoFalke) +- #18968 noban precludes maxuploadtarget disconnects (MarcoFalke) +- #19005 Add documentation for 'checklevel' argument in 'verifychain' RPC… (kcalvinalvin) +- #19192 Extract net permissions doc (MarcoFalke) +- #19071 Separate repository for the gui (MarcoFalke) +- #19018 fixing description of the field sequence in walletcreatefundedpsbt RPC method (limpbrains) +- #19367 Span pitfalls (sipa) +- #19408 Windows WSL build recommendation to temporarily disable Win32 PE support (sipsorcery) +- #19407 explain why passing -mlinker-version is required when cross-compiling (fanquake) +- #19452 afl fuzzing comment about afl-gcc and afl-g++ (Crypt-iQ) +- #19258 improve subtree check instructions (Sjors) +- #19474 Use precise permission flags where possible (MarcoFalke) +- #19494 CONTRIBUTING.md improvements (jonatack) +- #19268 Add non-thread-safe note to FeeFilterRounder::round() (hebasto) +- #19547 Update macOS cross compilation dependencies for Focal (hebasto) +- #19617 Clang 8 or later is required with `FORCE_USE_SYSTEM_CLANG` (fanquake) +- #19639 Remove Reference Links #19582 (RobertHosking) +- #19605 Set `CC_FOR_BUILD` when building on OpenBSD (fanquake) +- #19765 Fix getmempoolancestors RPC result doc (MarcoFalke) +- #19786 Remove label from good first issue template (MarcoFalke) +- #19646 Updated outdated help command for getblocktemplate (jakeleventhal) +- #18817 Document differences in bitcoind and bitcoin-qt locale handling (practicalswift) +- #19870 update PyZMQ install instructions, fix `zmq_sub.py` file permissions (jonatack) +- #19903 Update build-openbsd.md with GUI support (grubles) +- #19241 help: Generate checkpoint height from chainparams (luke-jr) +- #18949 Add CODEOWNERS file to automatically nominate PR reviewers (adamjonas) +- #20014 Mention signet in -help output (hebasto) +- #20015 Added default signet config for linearize script (gr0kchain) +- #19958 Better document features of feelers (naumenkogs) +- #19871 Clarify scope of eviction protection of outbound block-relay peers (ariard) +- #20076 Update and improve files.md (hebasto) +- #20107 Collect release-notes snippets (MarcoFalke) +- #20109 Release notes and followups from 19339 (glozow) +- #20090 Tiny followups to new getpeerinfo connection type field (amitiuttarwar) +- #20152 Update wallet files in files.md (hebasto) +- #19124 Document `ALLOW_HOST_PACKAGES` dependency option (skmcontrib) +- #20271 Document that wallet salvage is experimental (MarcoFalke) +- #20281 Correct getblockstats documentation for `(sw)total_weight` (shesek) +- #20279 release process updates/fixups (jonatack) +- #20238 Missing comments for signet parameters (decryp2kanon) +- #20756 Add missing field (permissions) to the getpeerinfo help (amitiuttarwar) +- #20668 warn that incoming conns are unlikely when not using default ports (adamjonas) +- #19961 tor.md updates (jonatack) +- #19050 Add warning for rest interface limitation (fjahr) +- #19390 doc/REST-interface: Remove stale info (luke-jr) +- #19344 docs: update testgen usage example (Bushstar) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 10xcryptodev +- Aaron Clauson +- Aaron Hook +- Adam Jonas +- Adam Soltys +- Adam Stein +- Akio Nakamura +- Alex Willmer +- Amir Ghorbanian +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Fieroni +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- Ben Carman +- Ben Woosley +- Benoit Verret +- Brian Liotti +- Bushstar +- Calvin Kim +- Carl Dong +- Chris Abrams +- Chris L +- Christopher Coverdale +- codeShark149 +- Cory Fields +- Craig Andrews +- Damian Mee +- Daniel Kraft +- Danny Lee +- David Reikher +- DesWurstes +- Dhruv Mehta +- Duncan Dean +- Elichai Turkel +- Elliott Jin +- Emil Engler +- Ethan Heilman +- eugene +- Fabian Jahr +- fanquake +- Ferdinando M. Ametrano +- freenancial +- furszy +- Gillian Chu +- Gleb Naumenko +- Glenn Willen +- Gloria Zhao +- glowang +- gr0kchain +- Gregory Sanders +- grubles +- gzhao408 +- Harris +- Hennadii Stepanov +- Hugo Nguyen +- Igor Cota +- Ivan Metlushko +- Ivan Vershigora +- Jake Leventhal +- James O'Beirne +- Jeremy Rubin +- jgmorgan +- Jim Posen +- “jkcd” +- jmorgan +- John Newbery +- Johnson Lau +- Jon Atack +- Jonas Schnelli +- Jonathan Schoeller +- João Barbosa +- Justin Moon +- kanon +- Karl-Johan Alm +- Kiminuo +- Kristaps Kaupe +- lontivero +- Luke Dashjr +- Marcin Jachymiak +- MarcoFalke +- Martin Ankerl +- Martin Zumsande +- maskoficarus +- Matt Corallo +- Matthew Zipkin +- MeshCollider +- Miguel Herranz +- MIZUTA Takeshi +- mruddy +- Nadav Ivgi +- Neha Narula +- Nicolas Thumann +- Niklas Gögge +- Nima Yazdanmehr +- nsa +- nthumann +- Oliver Gugger +- pad +- pasta +- Peter Bushnell +- pierrenn +- Pieter Wuille +- practicalswift +- Prayank +- Raúl Martínez (RME) +- RandyMcMillan +- Rene Pickhardt +- Riccardo Masutti +- Robert +- Rod Vagg +- Roy Shao +- Russell Yanofsky +- Saahil Shangle +- sachinkm77 +- saibato +- Samuel Dobson +- sanket1729 +- Sebastian Falbesoner +- Seleme Topuz +- Sishir Giri +- Sjors Provoost +- skmcontrib +- Stepan Snigirev +- Stephan Oeste +- Suhas Daftuar +- t-bast +- Tom Harding +- Torhte Butler +- TrentZ +- Troy Giorshev +- tryphe +- Tyler Chambers +- U-Zyn Chua +- Vasil Dimov +- wiz +- Wladimir J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 92845bcc82..84b208a0d8 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -5,7 +5,7 @@ Release Process ### Before every release candidate -* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). +* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). @@ -52,6 +52,13 @@ Release Process - Merge the release notes from the wiki into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. +#### Tagging a release (candidate) + +To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: + + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + +This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. ## Building @@ -73,21 +80,12 @@ Open a draft of the release notes for collaborative editing at https://github.co For the period during which the notes are being edited on the wiki, the version on the branch should be wiped and replaced with a link to the wiki which should be used for all announcements until `-final`. -Write the release notes. `git shortlog` helps a lot, for example: - - git shortlog --no-merges v(current version, e.g. 0.19.2)..v(new version, e.g. 0.20.0) - -(or ping @wumpus on IRC, he has specific tooling to generate the list of merged pulls -and sort them into categories based on labels). +Generate the change log. As this is a huge amount of work to do manually, there is the `list-pulls` script to do a pre-sorting step based on github PR metadata. See the [documentation in the README.md](https://github.com/bitcoin-core/bitcoin-maintainer-tools/blob/master/README.md#list-pulls). Generate list of authors: git log --format='- %aN' v(current version, e.g. 0.20.0)..v(new version, e.g. 0.20.1) | sort -fiu -Tag the version (or release candidate) in git: - - git tag -s v(new version, e.g. 0.20.0) - ### Setup and perform Gitian builds If you're using the automated script (found in [contrib/gitian-build.py](/contrib/gitian-build.py)), then at this point you should run it with the "--build" command. Otherwise ignore this. @@ -326,6 +324,18 @@ bitcoin.org (see below for bitcoin.org update instructions). - bitcoincore.org RPC documentation update + - Install [golang](https://golang.org/doc/install) + + - Install the new Bitcoin Core release + + - Run bitcoind on regtest + + - Clone the [bitcoincore.org repository](https://github.com/bitcoin-core/bitcoincore.org) + + - Run: `go run generate.go` while being in `contrib/doc-gen` folder, and with bitcoin-cli in PATH + + - Add the generated files to git + - Update packaging repo - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 diff --git a/doc/tor.md b/doc/tor.md index 1ba7137b8e..e38ada5bd6 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -21,129 +21,193 @@ information in the debug log about your Tor configuration. The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all outgoing connections, but more is possible. - -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy - server will be used to try to reach .onion addresses as well. + -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy + server will be used to try to reach .onion addresses as well. + You need to use -noonion or -onion=0 to explicitly disable + outbound access to onion services. + + -onion=ip:port Set the proxy server to use for Tor onion services. You do not + need to set this if it's the same as -proxy. You can use -onion=0 + to explicitly disable access to onion services. + Note: Only the -proxy option sets the proxy for DNS requests; + with -onion they will not route over Tor, so use -proxy if you + have privacy concerns. + + -listen When using -proxy, listening is disabled by default. If you want + to manually configure an onion service (see section 3), you'll + need to enable it explicitly. + + -connect=X When behind a Tor proxy, you can specify .onion addresses instead + -addnode=X of IP addresses or hostnames in these parameters. It requires + -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with + other P2P nodes. + + -onlynet=onion Make outgoing connections only to .onion addresses. Incoming + connections are not affected by this option. This option can be + specified multiple times to allow multiple network types, e.g. + ipv4, ipv6 or onion. If you use this option with values other + than onion you *cannot* disable onion connections; outgoing onion + connections will be enabled when you use -proxy or -onion. Use + -noonion or -onion=0 if you want to be sure there are no outbound + onion connections over the default proxy or your defined -proxy. - -onion=ip:port Set the proxy server to use for Tor onion services. You do not - need to set this if it's the same as -proxy. You can use -noonion - to explicitly disable access to onion services. +In a typical situation, this suffices to run behind a Tor proxy: - -listen When using -proxy, listening is disabled by default. If you want - to run an onion service (see next section), you'll need to enable - it explicitly. + ./bitcoind -proxy=127.0.0.1:9050 - -connect=X When behind a Tor proxy, you can specify .onion addresses instead - -addnode=X of IP addresses or hostnames in these parameters. It requires - -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with - other P2P nodes. +## 2. Automatically create a Bitcoin Core onion service - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - ipv4, ipv6, or onion. +Bitcoin Core makes use of Tor's control socket API to create and destroy +ephemeral onion services programmatically. This means that if Tor is running and +proper authentication has been configured, Bitcoin Core automatically creates an +onion service to listen on. The goal is to increase the number of available +onion nodes. -In a typical situation, this suffices to run behind a Tor proxy: +This feature is enabled by default if Bitcoin Core is listening (`-listen`) and +it requires a Tor connection to work. It can be explicitly disabled with +`-listenonion=0`. If it is not disabled, it can be configured using the +`-torcontrol` and `-torpassword` settings. + +To see verbose Tor information in the bitcoind debug log, pass `-debug=tor`. + +### Control Port + +You may need to set up the Tor Control Port. On Linux distributions there may be +some or all of the following settings in `/etc/tor/torrc`, generally commented +out by default (if not, add them): + +``` +ControlPort 9051 +CookieAuthentication 1 +CookieAuthFileGroupReadable 1 +``` + +Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` +or `sudo systemctl restart tor` on most systemd-based systems, including recent +Debian and Ubuntu, or just restart the computer). + +On some systems (such as Arch Linux), you may also need to add the following +line: + +``` +DataDirectoryGroupReadable 1 +``` + +### Authentication + +Connecting to Tor's control socket API requires one of two authentication +methods to be configured: cookie authentication or bitcoind's `-torpassword` +configuration option. + +#### Cookie authentication + +For cookie authentication, the user running bitcoind must have read access to +the `CookieAuthFile` specified in the Tor configuration. In some cases this is +preconfigured and the creation of an onion service is automatic. Don't forget to +use the `-debug=tor` bitcoind configuration option to enable Tor debug logging. + +If a permissions problem is seen in the debug log, e.g. `tor: Authentication +cookie /run/tor/control.authcookie could not be opened (check permissions)`, it +can be resolved by adding both the user running Tor and the user running +bitcoind to the same Tor group and setting permissions appropriately. + +On Debian-derived systems, the Tor group will likely be `debian-tor` and one way +to verify could be to list the groups and grep for a "tor" group name: + +``` +getent group | cut -d: -f1 | grep -i tor +``` + +You can also check the group of the cookie file. On most Linux systems, the Tor +auth cookie will usually be `/run/tor/control.authcookie`: + +``` +stat -c '%G' /run/tor/control.authcookie +``` + +Once you have determined the `${TORGROUP}` and selected the `${USER}` that will +run bitcoind, run this as root: + +``` +usermod -a -G ${TORGROUP} ${USER} +``` + +Then restart the computer (or log out) and log in as the `${USER}` that will run +bitcoind. + +#### `torpassword` authentication + +For the `-torpassword=password` option, the password is the clear text form that +was used when generating the hashed password for the `HashedControlPassword` +option in the Tor configuration file. - ./bitcoind -proxy=127.0.0.1:9050 +The hashed password can be obtained with the command `tor --hash-password +password` (refer to the [Tor Dev +Manual](https://2019.www.torproject.org/docs/tor-manual.html.en) for more +details). -## 2. Manually create a Bitcoin Core onion service +## 3. Manually create a Bitcoin Core onion service -If you configure your Tor system accordingly, it is possible to make your node also -reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent -config file): *Needed for Tor version 0.2.7.0 and older versions of Tor only. For newer -versions of Tor see [Section 3](#3-automatically-listen-on-tor).* +You can also manually configure your node to be reachable from the Tor network. +Add these lines to your `/etc/tor/torrc` (or equivalent config file): - HiddenServiceDir /var/lib/tor/bitcoin-service/ - HiddenServicePort 8333 127.0.0.1:8334 + HiddenServiceDir /var/lib/tor/bitcoin-service/ + HiddenServicePort 8333 127.0.0.1:8334 The directory can be different of course, but virtual port numbers should be equal to your bitcoind's P2P listen port (8333 by default), and target addresses and ports should be equal to binding address and port for inbound Tor connections (127.0.0.1:8334 by default). - -externalip=X You can tell bitcoin about its publicly reachable addresses using - this option, and this can be an onion address. Given the above - configuration, you can find your onion address in - /var/lib/tor/bitcoin-service/hostname. For connections - coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), onion addresses are given - preference for your node to advertise itself with. - - You can set multiple local addresses with -externalip. The - one that will be rumoured to a particular peer is the most - compatible one and also using heuristics, e.g. the address - with the most incoming connections, etc. - - -listen You'll need to enable listening for incoming connections, as this - is off by default behind a proxy. - - -discover When -externalip is specified, no attempt is made to discover local - IPv4 or IPv6 addresses. If you want to run a dual stack, reachable - from both Tor and IPv4 (or IPv6), you'll need to either pass your - other addresses using -externalip, or explicitly enable -discover. - Note that both addresses of a dual-stack system may be easily - linkable using traffic analysis. + -externalip=X You can tell bitcoin about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in + /var/lib/tor/bitcoin-service/hostname. For connections + coming from unroutable addresses (such as 127.0.0.1, where the + Tor proxy typically runs), onion addresses are given + preference for your node to advertise itself with. + + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + + -listen You'll need to enable listening for incoming connections, as this + is off by default behind a proxy. + + -discover When -externalip is specified, no attempt is made to discover local + IPv4 or IPv6 addresses. If you want to run a dual stack, reachable + from both Tor and IPv4 (or IPv6), you'll need to either pass your + other addresses using -externalip, or explicitly enable -discover. + Note that both addresses of a dual-stack system may be easily + linkable using traffic analysis. In a typical situation, where you're only reachable via Tor, this should suffice: - ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen + ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing your address. To mitigate this, additionally bind the address of your Tor proxy: - ./bitcoind ... -bind=127.0.0.1 + ./bitcoind ... -bind=127.0.0.1 If you don't care too much about hiding your node, and want to be reachable on IPv4 as well, use `discover` instead: - ./bitcoind ... -discover + ./bitcoind ... -discover and open port 8333 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover - -## 3. Automatically create a Bitcoin Core onion service - -Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket -API, to create and destroy 'ephemeral' onion services programmatically. -Bitcoin Core has been updated to make use of this. - -This means that if Tor is running (and proper authentication has been configured), -Bitcoin Core automatically creates an onion service to listen on. This will positively -affect the number of available .onion nodes. - -This new feature is enabled by default if Bitcoin Core is listening (`-listen`), and -requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0` -and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings. -To show verbose debugging information, pass `-debug=tor`. - -Connecting to Tor's control socket API requires one of two authentication methods to be -configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051` -in `torrc` config file. For cookie authentication the user running bitcoind must have read -access to the `CookieAuthFile` specified in Tor configuration. In some cases this is -preconfigured and the creation of an onion service is automatic. If permission problems -are seen with `-debug=tor` they can be resolved by adding both the user running Tor and -the user running bitcoind to the same group and setting permissions appropriately. On -Debian-based systems the user running bitcoind can be added to the debian-tor group, -which has the appropriate permissions. Before starting bitcoind you will need to re-login -to allow debian-tor group to be applied. Otherwise you will see the following notice: "tor: -Authentication cookie /run/tor/control.authcookie could not be opened (check permissions)" -on debug.log. - -An alternative authentication method is the use -of the `-torpassword=password` option. The `password` is the clear text form that -was used when generating the hashed password for the `HashedControlPassword` option -in the tor configuration file. The hashed password can be obtained with the command -`tor --hash-password password` (read the tor manual for more details). + ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover ## 4. Privacy recommendations -- Do not add anything but Bitcoin Core ports to the onion service created in section 2. +- Do not add anything but Bitcoin Core ports to the onion service created in section 3. If you run a web service too, create a new onion service for that. Otherwise it is trivial to link them, which may reduce privacy. Onion - services created automatically (as in section 3) always have only one port + services created automatically (as in section 2) always have only one port open. diff --git a/share/genbuild.sh b/share/genbuild.sh index d8f3429f7a..fe89ae1fde 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -31,7 +31,7 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ fi # otherwise generate suffix from git, i.e. string like "59887e8-dirty" - GIT_COMMIT=$(git rev-parse --short HEAD) + GIT_COMMIT=$(git rev-parse --short=12 HEAD) git diff-index --quiet HEAD -- || GIT_COMMIT="$GIT_COMMIT-dirty" fi diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 681f243d04..85ae7c57af 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -3,6 +3,7 @@ Name "@PACKAGE_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma SetDateSave off +Unicode true # Uncomment these lines when investigating reproducibility errors #SetCompress off diff --git a/src/.clang-format b/src/.clang-format index ef7a0ef5c7..a69c57f3e0 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -11,7 +11,8 @@ AllowShortIfStatementsOnASingleLine: true AllowShortLoopsOnASingleLine: false AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: true -BinPackParameters: false +BinPackArguments: true +BinPackParameters: true BreakBeforeBinaryOperators: false BreakBeforeBraces: Custom BraceWrapping: diff --git a/src/Makefile.am b/src/Makefile.am index 1a0791dccd..1c6d97b714 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,6 +2,10 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +# Pattern rule to print variables, e.g. make print-top_srcdir +print-%: + @echo '$*' = '$($*)' + DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) @@ -92,15 +96,21 @@ endif if BUILD_BITCOIN_CLI bin_PROGRAMS += bitcoin-cli endif + if BUILD_BITCOIN_TX bin_PROGRAMS += bitcoin-tx endif + if ENABLE_WALLET if BUILD_BITCOIN_WALLET bin_PROGRAMS += bitcoin-wallet endif endif +if BUILD_BITCOIN_UTIL + bin_PROGRAMS += bitcoin-util +endif + .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ @@ -138,6 +148,7 @@ BITCOIN_CORE_H = \ fs.h \ httprpc.h \ httpserver.h \ + i2p.h \ index/base.h \ index/blockfilterindex.h \ index/disktxpos.h \ @@ -221,21 +232,27 @@ BITCOIN_CORE_H = \ util/bip32.h \ util/bytevectorhash.h \ util/check.h \ + util/epochguard.h \ util/error.h \ util/fees.h \ + util/getuniquepath.h \ util/golombrice.h \ + util/hasher.h \ util/macros.h \ util/memory.h \ util/message.h \ util/moneystr.h \ util/rbf.h \ + util/readwritefile.h \ util/ref.h \ util/settings.h \ + util/sock.h \ util/spanparsing.h \ util/string.h \ util/system.h \ util/threadnames.h \ util/time.h \ + util/trace.h \ util/translation.h \ util/ui_change_type.h \ util/url.h \ @@ -251,10 +268,13 @@ BITCOIN_CORE_H = \ wallet/crypter.h \ wallet/db.h \ wallet/dump.h \ + wallet/external_signer.h \ + wallet/external_signer_scriptpubkeyman.h \ wallet/feebumper.h \ wallet/fees.h \ wallet/ismine.h \ wallet/load.h \ + wallet/rpcsigner.h \ wallet/rpcwallet.h \ wallet/salvage.h \ wallet/scriptpubkeyman.h \ @@ -296,6 +316,7 @@ libbitcoin_server_a_SOURCES = \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ + i2p.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ @@ -365,11 +386,14 @@ libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ wallet/dump.cpp \ + wallet/external_signer_scriptpubkeyman.cpp \ + wallet/external_signer.cpp \ wallet/feebumper.cpp \ wallet/fees.cpp \ wallet/interfaces.cpp \ wallet/load.cpp \ wallet/rpcdump.cpp \ + wallet/rpcsigner.cpp \ wallet/rpcwallet.cpp \ wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ @@ -544,10 +568,14 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/error.cpp \ util/fees.cpp \ + util/getuniquepath.cpp \ + util/hasher.cpp \ + util/sock.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ + util/readwritefile.cpp \ util/settings.cpp \ util/threadnames.cpp \ util/spanparsing.cpp \ @@ -666,6 +694,27 @@ bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc endif # +# bitcoin-util binary # +bitcoin_util_SOURCES = bitcoin-util.cpp +bitcoin_util_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_util_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_util_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + +if TARGET_WINDOWS +bitcoin_util_SOURCES += bitcoin-util-res.rc +endif + +bitcoin_util_LDADD = \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) + +bitcoin_util_LDADD += $(BOOST_LIBS) +# + # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 3d41d203d3..59cfdb9839 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -78,6 +78,7 @@ QT_MOC_CPP = \ qt/moc_transactiondesc.cpp \ qt/moc_transactiondescdialog.cpp \ qt/moc_transactionfilterproxy.cpp \ + qt/moc_transactionoverviewwidget.cpp \ qt/moc_transactiontablemodel.cpp \ qt/moc_transactionview.cpp \ qt/moc_utilitydialog.cpp \ @@ -151,6 +152,7 @@ BITCOIN_QT_H = \ qt/transactiondesc.h \ qt/transactiondescdialog.h \ qt/transactionfilterproxy.h \ + qt/transactionoverviewwidget.h \ qt/transactionrecord.h \ qt/transactiontablemodel.h \ qt/transactionview.h \ @@ -162,6 +164,9 @@ BITCOIN_QT_H = \ qt/walletview.h \ qt/winshutdownmonitor.h +RES_FONTS = \ + qt/res/fonts/RobotoMono-Bold.ttf + RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ @@ -288,7 +293,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_ANIMATION) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -359,7 +364,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_ANIMATION) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e9f9b73abe..e817bb2ee2 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,9 +2,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -if ENABLE_FUZZ +if ENABLE_FUZZ_BINARY noinst_PROGRAMS += test/fuzz/fuzz -else +endif + +if !ENABLE_FUZZ bin_PROGRAMS += test/test_bitcoin endif @@ -50,6 +52,14 @@ FUZZ_SUITE_LD_COMMON = \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) +if USE_UPNP +FUZZ_SUITE_LD_COMMON += $(MINIUPNPC_LIBS) +endif + +if USE_NATPMP +FUZZ_SUITE_LD_COMMON += $(NATPMP_LIBS) +endif + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -114,6 +124,7 @@ BITCOIN_TESTS =\ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ + test/sock_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ @@ -145,10 +156,16 @@ BITCOIN_TESTS += \ wallet/test/ismine_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp +FUZZ_SUITE_LD_COMMON +=\ + $(LIBBITCOIN_WALLET) \ + $(SQLITE_LIBS) \ + $(BDB_LIBS) + if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif + BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ @@ -172,12 +189,12 @@ test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $( if ENABLE_ZMQ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -if ENABLE_FUZZ - FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -278,8 +295,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/txrequest.cpp - -endif # ENABLE_FUZZ +endif # ENABLE_FUZZ_BINARY nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 4e858979fe..75fe68fcd1 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -16,6 +16,7 @@ libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAG libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ + test/fuzz/util.cpp \ $(TEST_FUZZ_H) LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 0621da8ddf..1abfb667a0 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -12,6 +12,7 @@ TEST_UTIL_H = \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ + test/util/script.h \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 8f77ed35ce..0922c1c432 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -109,15 +109,15 @@ template <typename Data> bool DeserializeFileDB(const fs::path& path, Data& data) { // open input file, and associate with CAutoFile - FILE *file = fsbridge::fopen(path, "rb"); + FILE* file = fsbridge::fopen(path, "rb"); CAutoFile filein(file, SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("%s: Failed to open file %s", __func__, path.string()); - + if (filein.IsNull()) { + LogPrintf("Missing or invalid file %s\n", path.string()); + return false; + } return DeserializeDB(filein, data); } - -} +} // namespace CBanDB::CBanDB(fs::path ban_list_path) : m_ban_list_path(std::move(ban_list_path)) { diff --git a/src/addrman.cpp b/src/addrman.cpp index ed7fccc0ff..f91121f156 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -9,6 +9,8 @@ #include <logging.h> #include <serialize.h> +#include <cmath> + int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); diff --git a/src/addrman.h b/src/addrman.h index 9ac67b7af6..92a5570953 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -335,22 +335,20 @@ public: * * nNew * * nTried * * number of "new" buckets XOR 2**30 - * * all nNew addrinfos in vvNew - * * all nTried addrinfos in vvTried - * * for each bucket: + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: * * number of elements - * * for each element: index + * * for each element: index in the serialized "all new addresses" + * * asmap checksum * * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it * as incompatible. This is necessary because it did not check the version number on * deserialization. * - * Notice that vvTried, mapAddr and vVector are never encoded explicitly; + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; * they are instead reconstructed from the other information. * - * vvNew is serialized, but only used if ADDRMAN_UNKNOWN_BUCKET_COUNT didn't change, - * otherwise it is reconstructed as well. - * * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * @@ -413,13 +411,13 @@ public: } } } - // Store asmap version after bucket entries so that it + // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_version; + uint256 asmap_checksum; if (m_asmap.size() != 0) { - asmap_version = SerializeHash(m_asmap); + asmap_checksum = SerializeHash(m_asmap); } - s << asmap_version; + s << asmap_checksum; } template <typename Stream> @@ -500,47 +498,63 @@ public: nTried -= nLost; // Store positions in the new table buckets to apply later (if possible). - std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing - - for (int bucket = 0; bucket < nUBuckets; bucket++) { - int nSize = 0; - s >> nSize; - for (int n = 0; n < nSize; n++) { - int nIndex = 0; - s >> nIndex; - if (nIndex >= 0 && nIndex < nNew) { - entryToBucket[nIndex] = bucket; + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector<std::pair<int, int>> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); } } } - uint256 supplied_asmap_version; + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum; if (m_asmap.size() != 0) { - supplied_asmap_version = SerializeHash(m_asmap); + supplied_asmap_checksum = SerializeHash(m_asmap); } - uint256 serialized_asmap_version; + uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_version; + s >> serialized_asmap_checksum; } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - int bucket = entryToBucket[n]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && - info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + CAddrInfo& info = mapInfo[entry_index]; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } else { - // In case the new table data cannot be used (format unknown, bucket count wrong or new asmap), + // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); bucket = info.GetNewBucket(nKey, m_asmap); - nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][nUBucketPos] == -1) { - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } } } diff --git a/src/banman.cpp b/src/banman.cpp index 49bf6c43dc..3fe561ad01 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -28,7 +28,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", m_banned.size(), GetTimeMillis() - n_start); } else { - LogPrintf("Invalid or missing banlist.dat; recreating\n"); + LogPrintf("Recreating banlist.dat\n"); SetBannedSetDirty(true); // force write DumpBanlist(); } diff --git a/src/base58.cpp b/src/base58.cpp index 65e373283c..fb04673c5c 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -52,7 +52,7 @@ static const int8_t mapBase58[256] = { int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up. std::vector<unsigned char> b256(size); // Process the characters. - static_assert(sizeof(mapBase58)/sizeof(mapBase58[0]) == 256, "mapBase58.size() should be 256"); // guarantee not out of range + static_assert(std::size(mapBase58) == 256, "mapBase58.size() should be 256"); // guarantee not out of range while (*psz && !IsSpace(*psz)) { // Decode base58 character int carry = mapBase58[(uint8_t)*psz]; diff --git a/src/bench/bench.h b/src/bench/bench.h index bafc7f8716..22f06d8cb8 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <util/macros.h> + #include <chrono> #include <functional> #include <map> @@ -12,8 +14,6 @@ #include <vector> #include <bench/nanobench.h> -#include <boost/preprocessor/cat.hpp> -#include <boost/preprocessor/stringize.hpp> /* * Usage: @@ -56,8 +56,8 @@ public: static void RunAll(const Args& args); }; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo"); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index af5a82f69f..8f656c44d9 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -48,9 +48,8 @@ static void AssembleBlock(benchmark::Bench& bench) LOCK(::cs_main); // Required for ::AcceptToMemoryPool. for (const auto& txr : txs) { - TxValidationState state; - bool ret{::AcceptToMemoryPool(*test_setup.m_node.mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */)}; - assert(ret); + const MempoolAcceptResult res = ::AcceptToMemoryPool(::ChainstateActive(), *test_setup.m_node.mempool, txr, false /* bypass_limits */); + assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index ffa772d8c1..d7b8c1badc 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -10,8 +10,6 @@ #include <random.h> #include <util/system.h> -#include <boost/thread/thread.hpp> - #include <vector> static const size_t BATCHES = 101; @@ -44,12 +42,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) void swap(PrevectorJob& x){p.swap(x.p);}; }; CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; - boost::thread_group tg; // The main thread should be counted to prevent thread oversubscription, and // to decrease the variance of benchmark results. - for (auto x = 0; x < GetNumCores() - 1; ++x) { - tg.create_thread([&]{queue.Thread();}); - } + queue.StartWorkerThreads(GetNumCores() - 1); // create all the data once, then submit copies in the benchmark. FastRandomContext insecure_rand(true); @@ -70,8 +65,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // it is done explicitly here for clarity control.Wait(); }); - tg.interrupt_all(); - tg.join_all(); + queue.StopWorkerThreads(); ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index e50c4476bb..3abfbfd784 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -42,20 +42,19 @@ static void CoinSelection(benchmark::Bench& bench) } addCoin(3 * COIN, wallet, wtxs); - // Create groups - std::vector<OutputGroup> groups; + // Create coins + std::vector<COutput> coins; for (const auto& wtx : wtxs) { - COutput output(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); - groups.emplace_back(output.GetInputCoin(), 6, false, 0, 0); + coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0); + const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0, false); bench.run([&] { std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool bnb_used; - bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used); + bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params, bnb_used); assert(success); assert(nValueRet == 1003 * COIN); assert(setCoinsRet.size() == 2); @@ -75,7 +74,8 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); - set.emplace_back(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0); + set.emplace_back(); + set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); wtxn.emplace_back(std::move(wtx)); } // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/data.cpp b/src/bench/data.cpp index 0ae4c7cad4..481e372105 100644 --- a/src/bench/data.cpp +++ b/src/bench/data.cpp @@ -8,7 +8,7 @@ namespace benchmark { namespace data { #include <bench/data/block413567.raw.h> -const std::vector<uint8_t> block413567{block413567_raw, block413567_raw + sizeof(block413567_raw) / sizeof(block413567_raw[0])}; +const std::vector<uint8_t> block413567{std::begin(block413567_raw), std::end(block413567_raw)}; } // namespace data } // namespace benchmark diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index dcd0e10285..f4fabedab6 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -76,7 +76,7 @@ static void PrevectorDeserialize(benchmark::Bench& bench) for (auto x = 0; x < 1000; ++x) { s0 >> t1; } - s0.Init(SER_NETWORK, 0); + s0.Rewind(); }); } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 4b45264a3c..78f8c6e6dc 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -7,27 +7,54 @@ #include <rpc/blockchain.h> #include <streams.h> +#include <test/util/setup_common.h> #include <validation.h> #include <univalue.h> -static void BlockToJsonVerbose(benchmark::Bench& bench) -{ - CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction +namespace { + +struct TestBlockAndIndex { + TestingSetup test_setup{}; + CBlock block{}; + uint256 blockHash{}; + CBlockIndex blockindex{}; + + TestBlockAndIndex() + { + CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); + char a = '\0'; + stream.write(&a, 1); // Prevent compaction - CBlock block; - stream >> block; + stream >> block; - CBlockIndex blockindex; - const uint256 blockHash = block.GetHash(); - blockindex.phashBlock = &blockHash; - blockindex.nBits = 403014710; + blockHash = block.GetHash(); + blockindex.phashBlock = &blockHash; + blockindex.nBits = 403014710; + } +}; +} // namespace + +static void BlockToJsonVerbose(benchmark::Bench& bench) +{ + TestBlockAndIndex data; bench.run([&] { - (void)blockToJSON(block, &blockindex, &blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + ankerl::nanobench::doNotOptimizeAway(univalue); }); } BENCHMARK(BlockToJsonVerbose); + +static void BlockToJsonVerboseWrite(benchmark::Bench& bench) +{ + TestBlockAndIndex data; + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + bench.run([&] { + auto str = univalue.write(); + ankerl::nanobench::doNotOptimizeAway(str); + }); +} + +BENCHMARK(BlockToJsonVerboseWrite); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 94043a6b45..0830cb54cb 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -21,6 +21,7 @@ #include <util/url.h> #include <algorithm> +#include <cmath> #include <functional> #include <memory> #include <stdio.h> @@ -57,7 +58,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC getnewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -299,9 +300,13 @@ class NetinfoRequestHandler : public BaseRequestHandler { private: static constexpr int8_t UNKNOWN_NETWORK{-1}; - static constexpr uint8_t m_networks_size{3}; - const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion"}}; - std::array<std::array<uint16_t, m_networks_size + 2>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total/block-relay) + static constexpr int8_t NET_I2P{3}; // pos of "i2p" in m_networks + static constexpr uint8_t m_networks_size{4}; + static constexpr uint8_t MAX_DETAIL_LEVEL{4}; + const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion", "i2p"}}; + std::array<std::array<uint16_t, m_networks_size + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) + uint8_t m_block_relay_peers_count{0}; + uint8_t m_manual_peers_count{0}; int8_t NetworkStringToId(const std::string& str) const { for (uint8_t i = 0; i < m_networks_size; ++i) { @@ -315,12 +320,14 @@ private: bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } bool m_is_asmap_on{false}; + bool m_is_i2p_on{false}; size_t m_max_addr_length{0}; - size_t m_max_age_length{4}; + size_t m_max_age_length{3}; size_t m_max_id_length{2}; struct Peer { std::string addr; std::string sub_version; + std::string conn_type; std::string network; std::string age; double min_ping; @@ -332,6 +339,8 @@ private: int id; int mapped_as; int version; + bool is_bip152_hb_from; + bool is_bip152_hb_to; bool is_block_relay; bool is_outbound; bool operator<(const Peer& rhs) const { return std::tie(is_outbound, min_ping) < std::tie(rhs.is_outbound, rhs.min_ping); } @@ -350,6 +359,14 @@ private: const double milliseconds{round(1000 * seconds)}; return milliseconds > 999999 ? "-" : ToString(milliseconds); } + std::string ConnectionTypeForNetinfo(const std::string& conn_type) const + { + if (conn_type == "outbound-full-relay") return "full"; + if (conn_type == "block-relay-only") return "block"; + if (conn_type == "manual" || conn_type == "feeler") return conn_type; + if (conn_type == "addr-fetch") return "addr"; + return ""; + } const UniValue NetinfoHelp() { return std::string{ @@ -378,6 +395,9 @@ private: " type Type of peer connection\n" " \"full\" - full relay, the default\n" " \"block\" - block relay; like full relay but does not relay transactions or addresses\n" + " \"manual\" - peer we manually added using RPC addnode or the -addnode/-connect config options\n" + " \"feeler\" - short-lived connection for testing addresses\n" + " \"addr\" - address fetch; short-lived connection for requesting addresses\n" " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", or \"cjdns\")\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" @@ -385,6 +405,9 @@ private: " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" + " hb High-bandwidth BIP152 compact block relay\n" + " \".\" (to) - we selected the peer as a high-bandwidth peer\n" + " \"*\" (from) - the peer selected us as a high-bandwidth peer\n" " age Duration of connection to the peer, in minutes\n" " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" @@ -392,7 +415,7 @@ private: " address IP address and port of the peer\n" " version Peer version and subversion concatenated, e.g. \"70016/Satoshi:21.0.0/\"\n\n" "* The connection counts table displays the number of peers by direction, network, and the totals\n" - " for each, as well as a column for block relay peers.\n\n" + " for each, as well as two special outbound columns for block relay peers and manual peers.\n\n" "* The local addresses table lists each local address broadcast by the node, the port, and the score.\n\n" "Examples:\n\n" "Connection counts and local addresses only\n" @@ -417,7 +440,7 @@ public: if (!args.empty()) { uint8_t n{0}; if (ParseUInt8(args.at(0), &n)) { - m_details_level = n; + m_details_level = std::min(n, MAX_DETAIL_LEVEL); } else if (args.at(0) == "help") { m_is_help_requested = true; } else { @@ -449,16 +472,16 @@ public: const std::string network{peer["network"].get_str()}; const int8_t network_id{NetworkStringToId(network)}; if (network_id == UNKNOWN_NETWORK) continue; + m_is_i2p_on |= (network_id == NET_I2P); const bool is_outbound{!peer["inbound"].get_bool()}; const bool is_block_relay{!peer["relaytxes"].get_bool()}; + const std::string conn_type{peer["connection_type"].get_str()}; ++m_counts.at(is_outbound).at(network_id); // in/out by network ++m_counts.at(is_outbound).at(m_networks_size); // in/out overall ++m_counts.at(2).at(network_id); // total by network ++m_counts.at(2).at(m_networks_size); // total overall - if (is_block_relay) { - ++m_counts.at(is_outbound).at(m_networks_size + 1); // in/out block-relay - ++m_counts.at(2).at(m_networks_size + 1); // total block-relay - } + if (conn_type == "block-relay-only") ++m_block_relay_peers_count; + if (conn_type == "manual") ++m_manual_peers_count; if (DetailsRequested()) { // Push data for this peer to the peers vector. const int peer_id{peer["id"].get_int()}; @@ -474,7 +497,9 @@ public: const std::string addr{peer["addr"].get_str()}; const std::string age{conn_time == 0 ? "" : ToString((m_time_now - conn_time) / 60)}; const std::string sub_version{peer["subver"].get_str()}; - m_peers.push_back({addr, sub_version, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_block_relay, is_outbound}); + const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; + const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; + m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_bip152_hb_from, is_bip152_hb_to, is_block_relay, is_outbound}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_age_length = std::max(age.length(), m_max_age_length); m_max_id_length = std::max(ToString(peer_id).length(), m_max_id_length); @@ -488,15 +513,15 @@ public: // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("<-> relay net mping ping send recv txn blk %*s ", m_max_age_length, "age"); + result += strprintf("<-> type net mping ping send recv txn blk hb %*s ", m_max_age_length, "age"); if (m_is_asmap_on) result += " asmap "; result += strprintf("%*s %-*s%s\n", m_max_id_length, "id", IsAddressSelected() ? m_max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); for (const Peer& peer : m_peers) { std::string version{ToString(peer.version) + peer.sub_version}; result += strprintf( - "%3s %5s %5s%7s%7s%5s%5s%5s%5s %*s%*i %*s %-*s%s\n", + "%3s %6s %5s%7s%7s%5s%5s%5s%5s %2s %*s%*i %*s %-*s%s\n", peer.is_outbound ? "out" : "in", - peer.is_block_relay ? "block" : "full", + ConnectionTypeForNetinfo(peer.conn_type), peer.network, PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), @@ -504,6 +529,7 @@ public: peer.last_recv == 0 ? "" : ToString(m_time_now - peer.last_recv), peer.last_trxn == 0 ? "" : ToString((m_time_now - peer.last_trxn) / 60), peer.last_blck == 0 ? "" : ToString((m_time_now - peer.last_blck) / 60), + strprintf("%s%s", peer.is_bip152_hb_to ? "." : " ", peer.is_bip152_hb_from ? "*" : " "), m_max_age_length, // variable spacing peer.age, m_is_asmap_on ? 7 : 0, // variable spacing @@ -514,18 +540,27 @@ public: IsAddressSelected() ? peer.addr : "", IsVersionSelected() && version != "0" ? version : ""); } - result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); + result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); } // Report peer connection totals by type. - result += " ipv4 ipv6 onion total block-relay\n"; + result += " ipv4 ipv6 onion"; + if (m_is_i2p_on) result += " i2p"; + result += " total block"; + if (m_manual_peers_count) result += " manual"; const std::array<std::string, 3> rows{{"in", "out", "total"}}; - for (uint8_t i = 0; i < m_networks_size; ++i) { - result += strprintf("%-5s %5i %5i %5i %5i %5i\n", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2), m_counts.at(i).at(m_networks_size), m_counts.at(i).at(m_networks_size + 1)); + for (uint8_t i = 0; i < 3; ++i) { + result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts + if (m_is_i2p_on) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + result += strprintf(" %5i", m_counts.at(i).at(m_networks_size)); // total peers count + if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts + result += strprintf(" %5i", m_block_relay_peers_count); + if (m_manual_peers_count) result += strprintf(" %5i", m_manual_peers_count); + } } // Report local addresses, ports, and scores. - result += "\nLocal addresses"; + result += "\n\nLocal addresses"; const std::vector<UniValue>& local_addrs{networkinfo["localaddresses"].getValues()}; if (local_addrs.empty()) { result += ": n/a\n"; diff --git a/src/bitcoin-util-res.rc b/src/bitcoin-util-res.rc new file mode 100644 index 0000000000..3f0fa8ab6d --- /dev/null +++ b/src/bitcoin-util-res.rc @@ -0,0 +1,35 @@ +#include <windows.h> // needed for VERSIONINFO +#include "clientversion.h" // holds the needed client version information + +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_FILEVERSION VER_PRODUCTVERSION +#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_APP +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" // U.S. English - multilingual (hex) + BEGIN + VALUE "CompanyName", "Bitcoin" + VALUE "FileDescription", "bitcoin-util (CLI Bitcoin utility)" + VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "InternalName", "bitcoin-util" + VALUE "LegalCopyright", COPYRIGHT_STR + VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." + VALUE "OriginalFilename", "bitcoin-util.exe" + VALUE "ProductName", "bitcoin-util" + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0, 1252 // language neutral - multilingual (decimal) + END +END diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp new file mode 100644 index 0000000000..af07b28d3d --- /dev/null +++ b/src/bitcoin-util.cpp @@ -0,0 +1,221 @@ +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <arith_uint256.h> +#include <clientversion.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <core_io.h> +#include <key_io.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <univalue.h> +#include <util/moneystr.h> +#include <util/rbf.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/system.h> +#include <util/translation.h> + +#include <atomic> +#include <functional> +#include <memory> +#include <stdio.h> +#include <thread> + +#include <boost/algorithm/string.hpp> + +static const int CONTINUE_EXECUTION=-1; + +const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; + +static void SetupBitcoinUtilArgs(ArgsManager &argsman) +{ + SetupHelpOptions(argsman); + + argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + + SetupChainParamsBaseOptions(argsman); +} + +// This function returns either one of EXIT_ codes when it's expected to stop the process or +// CONTINUE_EXECUTION when it's expected to continue further. +static int AppInitUtil(int argc, char* argv[]) +{ + SetupBitcoinUtilArgs(gArgs); + std::string error; + if (!gArgs.ParseParameters(argc, argv, error)) { + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); + return EXIT_FAILURE; + } + + // Check for chain settings (Params() calls are only valid after this clause) + try { + SelectParams(gArgs.GetChainName()); + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } + + if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + // First part of help message is specific to this utility + std::string strUsage = PACKAGE_NAME " bitcoin-util utility version " + FormatFullVersion() + "\n"; + if (!gArgs.IsArgSet("-version")) { + strUsage += "\n" + "Usage: bitcoin-util [options] [commands] Do stuff\n"; + strUsage += "\n" + gArgs.GetHelpMessage(); + } + + tfm::format(std::cout, "%s", strUsage); + + if (argc < 2) { + tfm::format(std::cerr, "Error: too few parameters\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; + } + return CONTINUE_EXECUTION; +} + +static void grind_task(uint32_t nBits, CBlockHeader& header_orig, uint32_t offset, uint32_t step, std::atomic<bool>& found) +{ + arith_uint256 target; + bool neg, over; + target.SetCompact(nBits, &neg, &over); + if (target == 0 || neg || over) return; + CBlockHeader header = header_orig; // working copy + header.nNonce = offset; + + uint32_t finish = std::numeric_limits<uint32_t>::max() - step; + finish = finish - (finish % step) + offset; + + while (!found && header.nNonce < finish) { + const uint32_t next = (finish - header.nNonce < 5000*step) ? finish : header.nNonce + 5000*step; + do { + if (UintToArith256(header.GetHash()) <= target) { + if (!found.exchange(true)) { + header_orig.nNonce = header.nNonce; + } + return; + } + header.nNonce += step; + } while(header.nNonce != next); + } +} + +static int Grind(int argc, char* argv[], std::string& strPrint) +{ + if (argc != 1) { + strPrint = "Must specify block header to grind"; + return 1; + } + + CBlockHeader header; + if (!DecodeHexBlockHeader(header, argv[0])) { + strPrint = "Could not decode block header"; + return 1; + } + + uint32_t nBits = header.nBits; + std::atomic<bool> found{false}; + + std::vector<std::thread> threads; + int n_tasks = std::max(1u, std::thread::hardware_concurrency()); + for (int i = 0; i < n_tasks; ++i) { + threads.emplace_back( grind_task, nBits, std::ref(header), i, n_tasks, std::ref(found) ); + } + for (auto& t : threads) { + t.join(); + } + if (!found) { + strPrint = "Could not satisfy difficulty target"; + return 1; + } + + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << header; + strPrint = HexStr(ss); + return 0; +} + +static int CommandLineUtil(int argc, char* argv[]) +{ + if (argc <= 1) return 1; + + std::string strPrint; + int nRet = 0; + + try { + while (argc > 1 && IsSwitchChar(argv[1][0]) && (argv[1][1] != 0)) { + --argc; + ++argv; + } + + char* command = argv[1]; + if (strcmp(command, "grind") == 0) { + nRet = Grind(argc-2, argv+2, strPrint); + } else { + strPrint = strprintf("Unknown command %s", command); + nRet = 1; + } + } + catch (const std::exception& e) { + strPrint = std::string("error: ") + e.what(); + nRet = EXIT_FAILURE; + } + catch (...) { + PrintExceptionContinue(nullptr, "CommandLineUtil()"); + throw; + } + + if (strPrint != "") { + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); + } + return nRet; +} + +#ifdef WIN32 +// Export main() and ensure working ASLR on Windows. +// Exporting a symbol will prevent the linker from stripping +// the .reloc section from the binary, which is a requirement +// for ASLR. This is a temporary workaround until a fixed +// version of binutils is used for releases. +__declspec(dllexport) int main(int argc, char* argv[]) +#else +int main(int argc, char* argv[]) +#endif +{ + SetupEnvironment(); + + try { + int ret = AppInitUtil(argc, argv); + if (ret != CONTINUE_EXECUTION) + return ret; + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "AppInitUtil()"); + return EXIT_FAILURE; + } catch (...) { + PrintExceptionContinue(nullptr, "AppInitUtil()"); + return EXIT_FAILURE; + } + + int ret = EXIT_FAILURE; + try { + ret = CommandLineUtil(argc, argv); + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "CommandLineUtil()"); + } catch (...) { + PrintExceptionContinue(nullptr, "CommandLineUtil()"); + } + return ret; +} diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 3e8e5fc7bc..b84d909b07 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -33,51 +33,52 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddCommand("info", "Get wallet info", OptionsCategory::COMMANDS); + argsman.AddCommand("create", "Create new wallet file", OptionsCategory::COMMANDS); + argsman.AddCommand("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", OptionsCategory::COMMANDS); + argsman.AddCommand("dump", "Print out all of the wallet key-value records", OptionsCategory::COMMANDS); + argsman.AddCommand("createfromdump", "Create new wallet file from dumped records", OptionsCategory::COMMANDS); } -static bool WalletAppInit(int argc, char* argv[]) +static bool WalletAppInit(ArgsManager& args, int argc, char* argv[]) { - SetupWalletToolArgs(gArgs); + SetupWalletToolArgs(args); std::string error_message; - if (!gArgs.ParseParameters(argc, argv, error_message)) { + if (!args.ParseParameters(argc, argv, error_message)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (argc < 2 || HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; - if (!gArgs.IsArgSet("-version")) { - strUsage += "\n" - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" - "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" - "Usage:\n" - " bitcoin-wallet [options] <command>\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); - } + if (!args.IsArgSet("-version")) { + strUsage += "\n" + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n"; + strUsage += "\n" + args.GetHelpMessage(); + } tfm::format(std::cout, "%s", strUsage); return false; } // check for printtoconsole, allow -debug - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", args.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")); return false; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); return true; } int main(int argc, char* argv[]) { + ArgsManager& args = gArgs; #ifdef WIN32 util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); @@ -85,7 +86,7 @@ int main(int argc, char* argv[]) SetupEnvironment(); RandomInit(); try { - if (!WalletAppInit(argc, argv)) return EXIT_FAILURE; + if (!WalletAppInit(args, argc, argv)) return EXIT_FAILURE; } catch (const std::exception& e) { PrintExceptionContinue(&e, "WalletAppInit()"); return EXIT_FAILURE; @@ -94,33 +95,19 @@ int main(int argc, char* argv[]) return EXIT_FAILURE; } - std::string method {}; - for(int i = 1; i < argc; ++i) { - if (!IsSwitchChar(argv[i][0])) { - if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); - return EXIT_FAILURE; - } - method = argv[i]; - } - } - - if (method.empty()) { + const auto command = args.GetCommand(); + if (!command) { tfm::format(std::cerr, "No method provided. Run `bitcoin-wallet -help` for valid methods.\n"); return EXIT_FAILURE; } - - // A name must be provided when creating a file - if (method == "create" && !gArgs.IsArgSet("-wallet")) { - tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + if (command->args.size() != 0) { + tfm::format(std::cerr, "Error: Additional arguments provided (%s). Methods do not take arguments. Please refer to `-help`.\n", Join(command->args, ", ")); return EXIT_FAILURE; } - std::string name = gArgs.GetArg("-wallet", ""); - ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) { + if (!WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; } ECC_Stop(); diff --git a/src/chain.h b/src/chain.h index 43e8a39f36..04a5db5a17 100644 --- a/src/chain.h +++ b/src/chain.h @@ -163,14 +163,27 @@ public: //! Number of transactions in this block. //! Note: in a potential headers-first mode, this number cannot be relied upon + //! Note: this value is faked during UTXO snapshot load to ensure that + //! LoadBlockIndex() will load index entries for blocks that we lack data for. + //! @sa ActivateSnapshot unsigned int nTx{0}; //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. //! Change to 64-bit type when necessary; won't happen before 2030 + //! + //! Note: this value is faked during use of a UTXO snapshot because we don't + //! have the underlying block data available during snapshot load. + //! @sa AssumeutxoData + //! @sa ActivateSnapshot unsigned int nChainTx{0}; //! Verification status of this block. See enum BlockStatus + //! + //! Note: this value is modified to show BLOCK_OPT_WITNESS during UTXO snapshot + //! load to avoid the block index being spuriously rewound. + //! @sa RewindBlockIndex + //! @sa ActivateSnapshot uint32_t nStatus{0}; //! block header diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 88cf5ef0a8..16efffa6f0 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -8,9 +8,7 @@ #include <chainparamsseeds.h> #include <consensus/merkle.h> #include <hash.h> // for signet block challenge hash -#include <tinyformat.h> #include <util/system.h> -#include <util/strencodings.h> #include <versionbitsinfo.h> #include <assert.h> @@ -136,7 +134,7 @@ public: bech32_hrp = "bc"; - vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main)); + vFixedSeeds = std::vector<SeedSpec6>(std::begin(pnSeed6_main), std::end(pnSeed6_main)); fDefaultConsistencyChecks = false; fRequireStandard = true; @@ -161,6 +159,10 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + // TODO to be specified in a future patch. + }; + chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 0000000000000000000b9d2ec5a352ecba0592946514a92f14319dc2b367fc72 /* nTime */ 1603995752, @@ -237,7 +239,7 @@ public: bech32_hrp = "tb"; - vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_test, pnSeed6_test + ARRAYLEN(pnSeed6_test)); + vFixedSeeds = std::vector<SeedSpec6>(std::begin(pnSeed6_test), std::end(pnSeed6_test)); fDefaultConsistencyChecks = false; fRequireStandard = false; @@ -250,6 +252,10 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + // TODO to be specified in a future patch. + }; + chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 000000000000006433d1efec504c53ca332b64963c425395515b01977bd7b3b0 /* nTime */ 1603359686, @@ -406,7 +412,7 @@ public: pchMessageStart[2] = 0xb5; pchMessageStart[3] = 0xda; nDefaultPort = 18444; - nPruneAfterHeight = 1000; + nPruneAfterHeight = gArgs.GetBoolArg("-fastprune", false) ? 100 : 1000; m_assumed_blockchain_size = 0; m_assumed_chain_state_size = 0; @@ -431,6 +437,17 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + { + 110, + {uint256S("0x76fd7334ac7c1baf57ddc0c626f073a655a35d98a4258cd1382c8cc2b8392e10"), 110}, + }, + { + 210, + {uint256S("0x9c5ed99ef98544b34f8920b6d1802f72ac28ae6e2bd2bd4c316ff10c230df3f2"), 210}, + }, + }; + chainTxData = ChainTxData{ 0, 0, @@ -526,3 +543,9 @@ void SelectParams(const std::string& network) SelectBaseParams(network); globalChainParams = CreateChainParams(gArgs, network); } + +std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud) +{ + o << strprintf("AssumeutxoData(%s, %s)", aud.hash_serialized.ToString(), aud.nChainTx); + return o; +} diff --git a/src/chainparams.h b/src/chainparams.h index d8b25c7220..4d24dcdb7c 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -31,6 +31,26 @@ struct CCheckpointData { }; /** + * Holds configuration for use during UTXO snapshot load and validation. The contents + * here are security critical, since they dictate which UTXO snapshots are recognized + * as valid. + */ +struct AssumeutxoData { + //! The expected hash of the deserialized UTXO set. + const uint256 hash_serialized; + + //! Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex(). + //! + //! We need to hardcode the value here because this is computed cumulatively using block data, + //! which we do not necessarily have at the time of snapshot load. + const unsigned int nChainTx; +}; + +std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud); + +using MapAssumeutxo = std::map<int, const AssumeutxoData>; + +/** * Holds various statistics on transactions within a chain. Used to estimate * verification progress during chain sync. * @@ -90,6 +110,11 @@ public: const std::string& Bech32HRP() const { return bech32_hrp; } const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } + + //! Get allowed assumeutxo configuration. + //! @see ChainstateManager + const MapAssumeutxo& Assumeutxo() const { return m_assumeutxo_data; } + const ChainTxData& TxData() const { return chainTxData; } protected: CChainParams() {} @@ -111,6 +136,7 @@ protected: bool m_is_test_chain; bool m_is_mockable_chain; CCheckpointData checkpointData; + MapAssumeutxo m_assumeutxo_data; ChainTxData chainTxData; }; diff --git a/src/checkqueue.h b/src/checkqueue.h index e3faa1dec0..4ceeb3600a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -6,13 +6,12 @@ #define BITCOIN_CHECKQUEUE_H #include <sync.h> +#include <tinyformat.h> +#include <util/threadnames.h> #include <algorithm> #include <vector> -#include <boost/thread/condition_variable.hpp> -#include <boost/thread/mutex.hpp> - template <typename T> class CCheckQueueControl; @@ -31,61 +30,64 @@ class CCheckQueue { private: //! Mutex to protect the inner state - boost::mutex mutex; + Mutex m_mutex; //! Worker threads block on this when out of work - boost::condition_variable condWorker; + std::condition_variable m_worker_cv; //! Master thread blocks on this when out of work - boost::condition_variable condMaster; + std::condition_variable m_master_cv; //! The queue of elements to be processed. //! As the order of booleans doesn't matter, it is used as a LIFO (stack) - std::vector<T> queue; + std::vector<T> queue GUARDED_BY(m_mutex); //! The number of workers (including the master) that are idle. - int nIdle; + int nIdle GUARDED_BY(m_mutex){0}; //! The total number of workers (including the master). - int nTotal; + int nTotal GUARDED_BY(m_mutex){0}; //! The temporary evaluation result. - bool fAllOk; + bool fAllOk GUARDED_BY(m_mutex){true}; /** * Number of verifications that haven't completed yet. * This includes elements that are no longer queued, but still in the * worker's own batches. */ - unsigned int nTodo; + unsigned int nTodo GUARDED_BY(m_mutex){0}; //! The maximum number of elements to be processed in one batch - unsigned int nBatchSize; + const unsigned int nBatchSize; + + std::vector<std::thread> m_worker_threads; + bool m_request_stop GUARDED_BY(m_mutex){false}; /** Internal function that does bulk of the verification work. */ - bool Loop(bool fMaster = false) + bool Loop(bool fMaster) { - boost::condition_variable& cond = fMaster ? condMaster : condWorker; + std::condition_variable& cond = fMaster ? m_master_cv : m_worker_cv; std::vector<T> vChecks; vChecks.reserve(nBatchSize); unsigned int nNow = 0; bool fOk = true; do { { - boost::unique_lock<boost::mutex> lock(mutex); + WAIT_LOCK(m_mutex, lock); // first do the clean-up of the previous loop run (allowing us to do it in the same critsect) if (nNow) { fAllOk &= fOk; nTodo -= nNow; if (nTodo == 0 && !fMaster) // We processed the last element; inform the master it can exit and return the result - condMaster.notify_one(); + m_master_cv.notify_one(); } else { // first iteration nTotal++; } // logically, the do loop starts here - while (queue.empty()) { + while (queue.empty() && !m_request_stop) { if (fMaster && nTodo == 0) { nTotal--; bool fRet = fAllOk; @@ -98,6 +100,10 @@ private: cond.wait(lock); // wait nIdle--; } + if (m_request_stop) { + return false; + } + // Decide how many work units to process now. // * Do not try to do everything at once, but aim for increasingly smaller batches so // all workers finish approximately simultaneously. @@ -106,7 +112,7 @@ private: nNow = std::max(1U, std::min(nBatchSize, (unsigned int)queue.size() / (nTotal + nIdle + 1))); vChecks.resize(nNow); for (unsigned int i = 0; i < nNow; i++) { - // We want the lock on the mutex to be as short as possible, so swap jobs from the global + // We want the lock on the m_mutex to be as short as possible, so swap jobs from the global // queue to the local batch vector instead of copying. vChecks[i].swap(queue.back()); queue.pop_back(); @@ -124,40 +130,68 @@ private: public: //! Mutex to ensure only one concurrent CCheckQueueControl - boost::mutex ControlMutex; + Mutex m_control_mutex; //! Create a new check queue - explicit CCheckQueue(unsigned int nBatchSizeIn) : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), nBatchSize(nBatchSizeIn) {} + explicit CCheckQueue(unsigned int nBatchSizeIn) + : nBatchSize(nBatchSizeIn) + { + } - //! Worker thread - void Thread() + //! Create a pool of new worker threads. + void StartWorkerThreads(const int threads_num) { - Loop(); + { + LOCK(m_mutex); + nIdle = 0; + nTotal = 0; + fAllOk = true; + } + assert(m_worker_threads.empty()); + for (int n = 0; n < threads_num; ++n) { + m_worker_threads.emplace_back([this, n]() { + util::ThreadRename(strprintf("scriptch.%i", n)); + Loop(false /* worker thread */); + }); + } } //! Wait until execution finishes, and return whether all evaluations were successful. bool Wait() { - return Loop(true); + return Loop(true /* master thread */); } //! Add a batch of checks to the queue void Add(std::vector<T>& vChecks) { - boost::unique_lock<boost::mutex> lock(mutex); + LOCK(m_mutex); for (T& check : vChecks) { queue.push_back(T()); check.swap(queue.back()); } nTodo += vChecks.size(); if (vChecks.size() == 1) - condWorker.notify_one(); + m_worker_cv.notify_one(); else if (vChecks.size() > 1) - condWorker.notify_all(); + m_worker_cv.notify_all(); + } + + //! Stop all of the worker threads. + void StopWorkerThreads() + { + WITH_LOCK(m_mutex, m_request_stop = true); + m_worker_cv.notify_all(); + for (std::thread& t : m_worker_threads) { + t.join(); + } + m_worker_threads.clear(); + WITH_LOCK(m_mutex, m_request_stop = false); } ~CCheckQueue() { + assert(m_worker_threads.empty()); } }; @@ -181,7 +215,7 @@ public: { // passed queue is supposed to be unused, or nullptr if (pqueue != nullptr) { - ENTER_CRITICAL_SECTION(pqueue->ControlMutex); + ENTER_CRITICAL_SECTION(pqueue->m_control_mutex); } } @@ -205,7 +239,7 @@ public: if (!fDone) Wait(); if (pqueue != nullptr) { - LEAVE_CRITICAL_SECTION(pqueue->ControlMutex); + LEAVE_CRITICAL_SECTION(pqueue->m_control_mutex); } } }; diff --git a/src/clientversion.h b/src/clientversion.h index 2da909f829..0ed3f68094 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CLIENTVERSION_H #define BITCOIN_CLIENTVERSION_H +#include <util/macros.h> + #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H @@ -14,13 +16,6 @@ #error Client version information missing: version is not defined by bitcoin-config.h or in any other way #endif -/** - * Converts the parameter X to a string after macro replacement on X has been performed. - * Don't merge these into one macro! - */ -#define STRINGIZE(X) DO_STRINGIZE(X) -#define DO_STRINGIZE(X) #X - //! Copyright string used in Windows .rc files #define COPYRIGHT_STR "2009-" STRINGIZE(COPYRIGHT_YEAR) " " COPYRIGHT_HOLDERS_FINAL diff --git a/src/coins.cpp b/src/coins.cpp index 14f58e956c..d52851cadd 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -31,8 +31,6 @@ bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} - CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { @@ -99,6 +97,14 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } +void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { + cachedCoinsUsage += coin.DynamicMemoryUsage(); + cacheCoins.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(outpoint)), + std::forward_as_tuple(std::move(coin), CCoinsCacheEntry::DIRTY)); +} + void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); const uint256& txid = tx.GetHash(); diff --git a/src/coins.h b/src/coins.h index a3e241ac90..feb441fd6a 100644 --- a/src/coins.h +++ b/src/coins.h @@ -8,11 +8,11 @@ #include <compressor.h> #include <core_memusage.h> -#include <crypto/siphash.h> #include <memusage.h> #include <primitives/transaction.h> #include <serialize.h> #include <uint256.h> +#include <util/hasher.h> #include <assert.h> #include <stdint.h> @@ -20,6 +20,8 @@ #include <functional> #include <unordered_map> +class ChainstateManager; + /** * A UTXO entry. * @@ -82,33 +84,6 @@ public: } }; -class SaltedOutpointHasher -{ -private: - /** Salt */ - const uint64_t k0, k1; - -public: - SaltedOutpointHasher(); - - /** - * This *must* return size_t. With Boost 1.46 on 32-bit systems the - * unordered_map will behave unpredictably if the custom hasher returns a - * uint64_t, resulting in failures when syncing the chain (#4634). - * - * Having the hash noexcept allows libstdc++'s unordered_map to recalculate - * the hash during rehash, so it does not have to cache the value. This - * reduces node's memory by sizeof(size_t). The required recalculation has - * a slight performance penalty (around 1.6%), but this is compensated by - * memory savings of about 9% which allow for a larger dbcache setting. - * - * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html - */ - size_t operator()(const COutPoint& id) const noexcept { - return SipHashUint256Extra(k0, k1, id.hash, id.n); - } -}; - /** * A Coin in one level of the coins database caching hierarchy. * @@ -152,6 +127,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : flags(0) {} explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} + CCoinsCacheEntry(Coin&& coin_, unsigned char flag) : coin(std::move(coin_)), flags(flag) {} }; typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; @@ -290,6 +266,15 @@ public: void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite); /** + * Emplace a coin into cacheCoins without performing any checks, marking + * the emplaced coin as dirty. + * + * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot. + * @sa ChainstateManager::PopulateAndValidateSnapshot() + */ + void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin); + + /** * Spend a coin. Pass moveto in order to get the deleted data. * If no unspent output exists for the passed outpoint, this call * has no effect. diff --git a/src/compat.h b/src/compat.h index dad14748a2..3449bc2661 100644 --- a/src/compat.h +++ b/src/compat.h @@ -44,6 +44,7 @@ typedef unsigned int SOCKET; #define WSAEINVAL EINVAL #define WSAEALREADY EALREADY #define WSAEWOULDBLOCK EWOULDBLOCK +#define WSAEAGAIN EAGAIN #define WSAEMSGSIZE EMSGSIZE #define WSAEINTR EINTR #define WSAEINPROGRESS EINPROGRESS @@ -51,6 +52,14 @@ typedef unsigned int SOCKET; #define WSAENOTSOCK EBADF #define INVALID_SOCKET (SOCKET)(~0) #define SOCKET_ERROR -1 +#else +#ifndef WSAEAGAIN +#ifdef EAGAIN +#define WSAEAGAIN EAGAIN +#else +#define WSAEAGAIN WSAEWOULDBLOCK +#endif +#endif #endif #ifdef WIN32 @@ -96,4 +105,14 @@ bool static inline IsSelectableSocket(const SOCKET& s) { #endif } +// MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 +#if !defined(MSG_NOSIGNAL) +#define MSG_NOSIGNAL 0 +#endif + +// MSG_DONTWAIT is not available on some platforms, if it doesn't exist define it as 0 +#if !defined(MSG_DONTWAIT) +#define MSG_DONTWAIT 0 +#endif + #endif // BITCOIN_COMPAT_H diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 301c2d914c..5f50cde3ff 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -17,16 +17,12 @@ # error "Bitcoin cannot be compiled without assertions." #endif -// Assumption: We assume a C++11 (ISO/IEC 14882:2011) compiler (minimum requirement). -// Example(s): We assume the presence of C++11 features everywhere :-) -// Note: MSVC does not report the expected __cplusplus value due to legacy -// reasons. -#if !defined(_MSC_VER) -// ISO Standard C++11 [cpp.predefined]p1: -// "The name __cplusplus is defined to the value 201103L when compiling a C++ +// Assumption: We assume a C++17 (ISO/IEC 14882:2017) compiler (minimum requirement). +// Example(s): We assume the presence of C++17 features everywhere :-) +// ISO Standard C++17 [cpp.predefined]p1: +// "The name __cplusplus is defined to the value 201703L when compiling a C++ // translation unit." -static_assert(__cplusplus >= 201103L, "C++11 standard assumed"); -#endif +static_assert(__cplusplus >= 201703L, "C++17 standard assumed"); // Assumption: We assume the floating-point types to fulfill the requirements of // IEC 559 (IEEE 754) standard. diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 6d7a293f9b..8a51f310f7 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -54,6 +54,12 @@ __asm(".symver log2f_old,log2f@GLIBC_2.2.5"); __asm(".symver log2f_old,log2f@GLIBC_2.4"); #elif defined(__aarch64__) __asm(".symver log2f_old,log2f@GLIBC_2.17"); +#elif defined(__powerpc64__) +# ifdef WORDS_BIGENDIAN +__asm(".symver log2f_old,log2f@GLIBC_2.3"); +# else +__asm(".symver log2f_old,log2f@GLIBC_2.17"); +# endif #elif defined(__riscv) __asm(".symver log2f_old,log2f@GLIBC_2.27"); #endif diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index fbd14f9325..e5a0d4cb9c 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -17,7 +17,6 @@ namespace { using limb_t = Num3072::limb_t; using double_limb_t = Num3072::double_limb_t; constexpr int LIMB_SIZE = Num3072::LIMB_SIZE; -constexpr int LIMBS = Num3072::LIMBS; /** 2^3072 - 1103717, the largest 3072-bit safe prime number, is used as the modulus. */ constexpr limb_t MAX_PRIME_DIFF = 1103717; @@ -123,7 +122,7 @@ inline void square_n_mul(Num3072& in_out, const int sq, const Num3072& mul) } // namespace -/** Indicates wether d is larger than the modulus. */ +/** Indicates whether d is larger than the modulus. */ bool Num3072::IsOverflow() const { if (this->limbs[0] <= std::numeric_limits<limb_t>::max() - MAX_PRIME_DIFF) return false; @@ -276,18 +275,33 @@ void Num3072::Divide(const Num3072& a) if (this->IsOverflow()) this->FullReduce(); } -Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { - Num3072 out{}; - uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); - unsigned char tmp[BYTE_SIZE]; - ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, BYTE_SIZE); +Num3072::Num3072(const unsigned char (&data)[BYTE_SIZE]) { + for (int i = 0; i < LIMBS; ++i) { + if (sizeof(limb_t) == 4) { + this->limbs[i] = ReadLE32(data + 4 * i); + } else if (sizeof(limb_t) == 8) { + this->limbs[i] = ReadLE64(data + 8 * i); + } + } +} + +void Num3072::ToBytes(unsigned char (&out)[BYTE_SIZE]) { for (int i = 0; i < LIMBS; ++i) { if (sizeof(limb_t) == 4) { - out.limbs[i] = ReadLE32(tmp + 4 * i); + WriteLE32(out + i * 4, this->limbs[i]); } else if (sizeof(limb_t) == 8) { - out.limbs[i] = ReadLE64(tmp + 8 * i); + WriteLE64(out + i * 8, this->limbs[i]); } } +} + +Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { + unsigned char tmp[Num3072::BYTE_SIZE]; + + uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); + ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, Num3072::BYTE_SIZE); + Num3072 out{tmp}; + return out; } @@ -301,14 +315,8 @@ void MuHash3072::Finalize(uint256& out) noexcept m_numerator.Divide(m_denominator); m_denominator.SetToOne(); // Needed to keep the MuHash object valid - unsigned char data[384]; - for (int i = 0; i < LIMBS; ++i) { - if (sizeof(limb_t) == 4) { - WriteLE32(data + i * 4, m_numerator.limbs[i]); - } else if (sizeof(limb_t) == 8) { - WriteLE64(data + i * 8, m_numerator.limbs[i]); - } - } + unsigned char data[Num3072::BYTE_SIZE]; + m_numerator.ToBytes(data); out = (CHashWriter(SER_DISK, 0) << data).GetSHA256(); } diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h index 0c710007c4..c023a8b9d3 100644 --- a/src/crypto/muhash.h +++ b/src/crypto/muhash.h @@ -22,6 +22,7 @@ private: Num3072 GetInverse() const; public: + static constexpr size_t BYTE_SIZE = 384; #ifdef HAVE___INT128 typedef unsigned __int128 double_limb_t; @@ -48,8 +49,10 @@ public: void Divide(const Num3072& a); void SetToOne(); void Square(); + void ToBytes(unsigned char (&out)[BYTE_SIZE]); Num3072() { this->SetToOne(); }; + Num3072(const unsigned char (&data)[BYTE_SIZE]); SERIALIZE_METHODS(Num3072, obj) { @@ -78,7 +81,7 @@ public: * arbitrary subset of the update operations, allowing them to be * efficiently combined later. * - * Muhash does not support checking if an element is already part of the + * MuHash does not support checking if an element is already part of the * set. That is why this class does not enforce the use of a set as the * data it represents because there is no efficient way to do so. * It is possible to add elements more than once and also to remove @@ -91,8 +94,6 @@ public: class MuHash3072 { private: - static constexpr size_t BYTE_SIZE = 384; - Num3072 m_numerator; Num3072 m_denominator; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 33e4b366a1..c119036db2 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -8,9 +8,10 @@ #include <clientversion.h> #include <fs.h> #include <serialize.h> +#include <span.h> #include <streams.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> @@ -73,12 +74,12 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue(ssValue.data(), ssValue.size()); + leveldb::Slice slValue((const char*)ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); // LevelDB serializes writes as: @@ -98,7 +99,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); batch.Delete(slKey); // LevelDB serializes erases as: @@ -137,7 +138,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); piter->Seek(slKey); } @@ -146,7 +147,7 @@ public: template<typename K> bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); try { - CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssKey(MakeUCharSpan(slKey), SER_DISK, CLIENT_VERSION); ssKey >> key; } catch (const std::exception&) { return false; @@ -157,7 +158,7 @@ public: template<typename V> bool GetValue(V& value) { leveldb::Slice slValue = piter->value(); try { - CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(slValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { @@ -232,7 +233,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -243,7 +244,7 @@ public: dbwrapper_private::HandleError(status); } try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(strValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { @@ -266,7 +267,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -310,8 +311,8 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); pdb->GetApproximateSizes(&range, 1, &size); @@ -329,11 +330,10 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); pdb->CompactRange(&slKey1, &slKey2); } - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 4543f098a1..bb06c95e7d 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -38,6 +38,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-paytxfee=<amt>", "-rescan", "-salvagewallet", + "-signer=<cmd>", "-spendzeroconfchange", "-txconfirmtarget=<n>", "-wallet=<path>", diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 11cf357f3d..151f1a38f1 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -66,7 +66,7 @@ size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_ if (CheckDiskSpace(m_dir, inc_size)) { FILE *file = Open(pos); if (file) { - LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + LogPrint(BCLog::VALIDATION, "Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); AllocateFileRange(file, pos.nPos, inc_size); fclose(file); return inc_size; diff --git a/src/fs.cpp b/src/fs.cpp index 7eb0f5ca9f..4f20ca4d28 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -31,6 +31,12 @@ FILE *fopen(const fs::path& p, const char *mode) #endif } +fs::path AbsPathJoin(const fs::path& base, const fs::path& path) +{ + assert(base.is_absolute()); + return fs::absolute(path, base); +} + #ifndef WIN32 static std::string GetErrorReason() @@ -21,6 +21,17 @@ namespace fs = boost::filesystem; namespace fsbridge { FILE *fopen(const fs::path& p, const char *mode); + /** + * Helper function for joining two paths + * + * @param[in] base Base path + * @param[in] path Path to combine with base + * @returns path unchanged if it is an absolute path, otherwise returns base joined with path. Returns base unchanged if path is empty. + * @pre Base path must be absolute + * @post Returned path will always be absolute + */ + fs::path AbsPathJoin(const fs::path& base, const fs::path& path); + class FileLock { public: diff --git a/src/i2p.cpp b/src/i2p.cpp new file mode 100644 index 0000000000..42270deaeb --- /dev/null +++ b/src/i2p.cpp @@ -0,0 +1,407 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compat.h> +#include <compat/endian.h> +#include <crypto/sha256.h> +#include <fs.h> +#include <i2p.h> +#include <logging.h> +#include <netaddress.h> +#include <netbase.h> +#include <random.h> +#include <util/strencodings.h> +#include <tinyformat.h> +#include <util/readwritefile.h> +#include <util/sock.h> +#include <util/spanparsing.h> +#include <util/system.h> + +#include <chrono> +#include <stdexcept> +#include <string> + +namespace i2p { + +/** + * Swap Standard Base64 <-> I2P Base64. + * Standard Base64 uses `+` and `/` as last two characters of its alphabet. + * I2P Base64 uses `-` and `~` respectively. + * So it is easy to detect in which one is the input and convert to the other. + * @param[in] from Input to convert. + * @return converted `from` + */ +static std::string SwapBase64(const std::string& from) +{ + std::string to; + to.resize(from.size()); + for (size_t i = 0; i < from.size(); ++i) { + switch (from[i]) { + case '-': + to[i] = '+'; + break; + case '~': + to[i] = '/'; + break; + case '+': + to[i] = '-'; + break; + case '/': + to[i] = '~'; + break; + default: + to[i] = from[i]; + break; + } + } + return to; +} + +/** + * Decode an I2P-style Base64 string. + * @param[in] i2p_b64 I2P-style Base64 string. + * @return decoded `i2p_b64` + * @throw std::runtime_error if decoding fails + */ +static Binary DecodeI2PBase64(const std::string& i2p_b64) +{ + const std::string& std_b64 = SwapBase64(i2p_b64); + bool invalid; + Binary decoded = DecodeBase64(std_b64.c_str(), &invalid); + if (invalid) { + throw std::runtime_error(strprintf("Cannot decode Base64: \"%s\"", i2p_b64)); + } + return decoded; +} + +/** + * Derive the .b32.i2p address of an I2P destination (binary). + * @param[in] dest I2P destination. + * @return the address that corresponds to `dest` + * @throw std::runtime_error if conversion fails + */ +static CNetAddr DestBinToAddr(const Binary& dest) +{ + CSHA256 hasher; + hasher.Write(dest.data(), dest.size()); + unsigned char hash[CSHA256::OUTPUT_SIZE]; + hasher.Finalize(hash); + + CNetAddr addr; + const std::string addr_str = EncodeBase32(hash, false) + ".b32.i2p"; + if (!addr.SetSpecial(addr_str)) { + throw std::runtime_error(strprintf("Cannot parse I2P address: \"%s\"", addr_str)); + } + + return addr; +} + +/** + * Derive the .b32.i2p address of an I2P destination (I2P-style Base64). + * @param[in] dest I2P destination. + * @return the address that corresponds to `dest` + * @throw std::runtime_error if conversion fails + */ +static CNetAddr DestB64ToAddr(const std::string& dest) +{ + const Binary& decoded = DecodeI2PBase64(dest); + return DestBinToAddr(decoded); +} + +namespace sam { + +Session::Session(const fs::path& private_key_file, + const CService& control_host, + CThreadInterrupt* interrupt) + : m_private_key_file(private_key_file), m_control_host(control_host), m_interrupt(interrupt) +{ +} + +Session::~Session() +{ + LOCK(m_mutex); + Disconnect(); +} + +bool Session::Listen(Connection& conn) +{ + try { + LOCK(m_mutex); + CreateIfNotCreatedAlready(); + conn.me = m_my_addr; + conn.sock = StreamAccept(); + return true; + } catch (const std::runtime_error& e) { + Log("Error listening: %s", e.what()); + CheckControlSock(); + } + return false; +} + +bool Session::Accept(Connection& conn) +{ + try { + while (!*m_interrupt) { + Sock::Event occurred; + conn.sock.Wait(MAX_WAIT_FOR_IO, Sock::RECV, &occurred); + + if ((occurred & Sock::RECV) == 0) { + // Timeout, no incoming connections within MAX_WAIT_FOR_IO. + continue; + } + + const std::string& peer_dest = + conn.sock.RecvUntilTerminator('\n', MAX_WAIT_FOR_IO, *m_interrupt); + + conn.peer = CService(DestB64ToAddr(peer_dest), Params().GetDefaultPort()); + + return true; + } + } catch (const std::runtime_error& e) { + Log("Error accepting: %s", e.what()); + CheckControlSock(); + } + return false; +} + +bool Session::Connect(const CService& to, Connection& conn, bool& proxy_error) +{ + proxy_error = true; + + std::string session_id; + Sock sock; + conn.peer = to; + + try { + { + LOCK(m_mutex); + CreateIfNotCreatedAlready(); + session_id = m_session_id; + conn.me = m_my_addr; + sock = Hello(); + } + + const Reply& lookup_reply = + SendRequestAndGetReply(sock, strprintf("NAMING LOOKUP NAME=%s", to.ToStringIP())); + + const std::string& dest = lookup_reply.Get("VALUE"); + + const Reply& connect_reply = SendRequestAndGetReply( + sock, strprintf("STREAM CONNECT ID=%s DESTINATION=%s SILENT=false", session_id, dest), + false); + + const std::string& result = connect_reply.Get("RESULT"); + + if (result == "OK") { + conn.sock = std::move(sock); + return true; + } + + if (result == "INVALID_ID") { + LOCK(m_mutex); + Disconnect(); + throw std::runtime_error("Invalid session id"); + } + + if (result == "CANT_REACH_PEER" || result == "TIMEOUT") { + proxy_error = false; + } + + throw std::runtime_error(strprintf("\"%s\"", connect_reply.full)); + } catch (const std::runtime_error& e) { + Log("Error connecting to %s: %s", to.ToString(), e.what()); + CheckControlSock(); + return false; + } +} + +// Private methods + +std::string Session::Reply::Get(const std::string& key) const +{ + const auto& pos = keys.find(key); + if (pos == keys.end() || !pos->second.has_value()) { + throw std::runtime_error( + strprintf("Missing %s= in the reply to \"%s\": \"%s\"", key, request, full)); + } + return pos->second.value(); +} + +template <typename... Args> +void Session::Log(const std::string& fmt, const Args&... args) const +{ + LogPrint(BCLog::I2P, "I2P: %s\n", tfm::format(fmt, args...)); +} + +Session::Reply Session::SendRequestAndGetReply(const Sock& sock, + const std::string& request, + bool check_result_ok) const +{ + sock.SendComplete(request + "\n", MAX_WAIT_FOR_IO, *m_interrupt); + + Reply reply; + + // Don't log the full "SESSION CREATE ..." because it contains our private key. + reply.request = request.substr(0, 14) == "SESSION CREATE" ? "SESSION CREATE ..." : request; + + // It could take a few minutes for the I2P router to reply as it is querying the I2P network + // (when doing name lookup, for example). Notice: `RecvUntilTerminator()` is checking + // `m_interrupt` more often, so we would not be stuck here for long if `m_interrupt` is + // signaled. + static constexpr auto recv_timeout = 3min; + + reply.full = sock.RecvUntilTerminator('\n', recv_timeout, *m_interrupt); + + for (const auto& kv : spanparsing::Split(reply.full, ' ')) { + const auto& pos = std::find(kv.begin(), kv.end(), '='); + if (pos != kv.end()) { + reply.keys.emplace(std::string{kv.begin(), pos}, std::string{pos + 1, kv.end()}); + } else { + reply.keys.emplace(std::string{kv.begin(), kv.end()}, std::nullopt); + } + } + + if (check_result_ok && reply.Get("RESULT") != "OK") { + throw std::runtime_error( + strprintf("Unexpected reply to \"%s\": \"%s\"", request, reply.full)); + } + + return reply; +} + +Sock Session::Hello() const +{ + auto sock = CreateSock(m_control_host); + + if (!sock) { + throw std::runtime_error("Cannot create socket"); + } + + if (!ConnectSocketDirectly(m_control_host, sock->Get(), nConnectTimeout, true)) { + throw std::runtime_error(strprintf("Cannot connect to %s", m_control_host.ToString())); + } + + SendRequestAndGetReply(*sock, "HELLO VERSION MIN=3.1 MAX=3.1"); + + return std::move(*sock); +} + +void Session::CheckControlSock() +{ + LOCK(m_mutex); + + std::string errmsg; + if (!m_control_sock.IsConnected(errmsg)) { + Log("Control socket error: %s", errmsg); + Disconnect(); + } +} + +void Session::DestGenerate(const Sock& sock) +{ + // https://geti2p.net/spec/common-structures#key-certificates + // "7" or "EdDSA_SHA512_Ed25519" - "Recent Router Identities and Destinations". + // Use "7" because i2pd <2.24.0 does not recognize the textual form. + const Reply& reply = SendRequestAndGetReply(sock, "DEST GENERATE SIGNATURE_TYPE=7", false); + + m_private_key = DecodeI2PBase64(reply.Get("PRIV")); +} + +void Session::GenerateAndSavePrivateKey(const Sock& sock) +{ + DestGenerate(sock); + + // umask is set to 077 in init.cpp, which is ok (unless -sysperms is given) + if (!WriteBinaryFile(m_private_key_file, + std::string(m_private_key.begin(), m_private_key.end()))) { + throw std::runtime_error( + strprintf("Cannot save I2P private key to %s", m_private_key_file)); + } +} + +Binary Session::MyDestination() const +{ + // From https://geti2p.net/spec/common-structures#destination: + // "They are 387 bytes plus the certificate length specified at bytes 385-386, which may be + // non-zero" + static constexpr size_t DEST_LEN_BASE = 387; + static constexpr size_t CERT_LEN_POS = 385; + + uint16_t cert_len; + memcpy(&cert_len, &m_private_key.at(CERT_LEN_POS), sizeof(cert_len)); + cert_len = be16toh(cert_len); + + const size_t dest_len = DEST_LEN_BASE + cert_len; + + return Binary{m_private_key.begin(), m_private_key.begin() + dest_len}; +} + +void Session::CreateIfNotCreatedAlready() +{ + std::string errmsg; + if (m_control_sock.IsConnected(errmsg)) { + return; + } + + Log("Creating SAM session with %s", m_control_host.ToString()); + + Sock sock = Hello(); + + const auto& [read_ok, data] = ReadBinaryFile(m_private_key_file); + if (read_ok) { + m_private_key.assign(data.begin(), data.end()); + } else { + GenerateAndSavePrivateKey(sock); + } + + const std::string& session_id = GetRandHash().GetHex().substr(0, 10); // full is an overkill, too verbose in the logs + const std::string& private_key_b64 = SwapBase64(EncodeBase64(m_private_key)); + + SendRequestAndGetReply(sock, strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=%s", + session_id, private_key_b64)); + + m_my_addr = CService(DestBinToAddr(MyDestination()), Params().GetDefaultPort()); + m_session_id = session_id; + m_control_sock = std::move(sock); + + LogPrintf("I2P: SAM session created: session id=%s, my address=%s\n", m_session_id, + m_my_addr.ToString()); +} + +Sock Session::StreamAccept() +{ + Sock sock = Hello(); + + const Reply& reply = SendRequestAndGetReply( + sock, strprintf("STREAM ACCEPT ID=%s SILENT=false", m_session_id), false); + + const std::string& result = reply.Get("RESULT"); + + if (result == "OK") { + return sock; + } + + if (result == "INVALID_ID") { + // If our session id is invalid, then force session re-creation on next usage. + Disconnect(); + } + + throw std::runtime_error(strprintf("\"%s\"", reply.full)); +} + +void Session::Disconnect() +{ + if (m_control_sock.Get() != INVALID_SOCKET) { + if (m_session_id.empty()) { + Log("Destroying incomplete session"); + } else { + Log("Destroying session %s", m_session_id); + } + } + m_control_sock.Reset(); + m_session_id.clear(); +} +} // namespace sam +} // namespace i2p diff --git a/src/i2p.h b/src/i2p.h new file mode 100644 index 0000000000..8fafe0a4d0 --- /dev/null +++ b/src/i2p.h @@ -0,0 +1,260 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_I2P_H +#define BITCOIN_I2P_H + +#include <compat.h> +#include <fs.h> +#include <netaddress.h> +#include <sync.h> +#include <threadinterrupt.h> +#include <util/sock.h> + +#include <optional> +#include <string> +#include <unordered_map> +#include <vector> + +namespace i2p { + +/** + * Binary data. + */ +using Binary = std::vector<uint8_t>; + +/** + * An established connection with another peer. + */ +struct Connection { + /** Connected socket. */ + Sock sock; + + /** Our I2P address. */ + CService me; + + /** The peer's I2P address. */ + CService peer; +}; + +namespace sam { + +/** + * I2P SAM session. + */ +class Session +{ +public: + /** + * Construct a session. This will not initiate any IO, the session will be lazily created + * later when first used. + * @param[in] private_key_file Path to a private key file. If the file does not exist then the + * private key will be generated and saved into the file. + * @param[in] control_host Location of the SAM proxy. + * @param[in,out] interrupt If this is signaled then all operations are canceled as soon as + * possible and executing methods throw an exception. Notice: only a pointer to the + * `CThreadInterrupt` object is saved, so it must not be destroyed earlier than this + * `Session` object. + */ + Session(const fs::path& private_key_file, + const CService& control_host, + CThreadInterrupt* interrupt); + + /** + * Destroy the session, closing the internally used sockets. The sockets that have been + * returned by `Accept()` or `Connect()` will not be closed, but they will be closed by + * the SAM proxy because the session is destroyed. So they will return an error next time + * we try to read or write to them. + */ + ~Session(); + + /** + * Start listening for an incoming connection. + * @param[out] conn Upon successful completion the `sock` and `me` members will be set + * to the listening socket and address. + * @return true on success + */ + bool Listen(Connection& conn); + + /** + * Wait for and accept a new incoming connection. + * @param[in,out] conn The `sock` member is used for waiting and accepting. Upon successful + * completion the `peer` member will be set to the address of the incoming peer. + * @return true on success + */ + bool Accept(Connection& conn); + + /** + * Connect to an I2P peer. + * @param[in] to Peer to connect to. + * @param[out] conn Established connection. Only set if `true` is returned. + * @param[out] proxy_error If an error occurs due to proxy or general network failure, then + * this is set to `true`. If an error occurs due to unreachable peer (likely peer is down), then + * it is set to `false`. Only set if `false` is returned. + * @return true on success + */ + bool Connect(const CService& to, Connection& conn, bool& proxy_error); + +private: + /** + * A reply from the SAM proxy. + */ + struct Reply { + /** + * Full, unparsed reply. + */ + std::string full; + + /** + * Request, used for detailed error reporting. + */ + std::string request; + + /** + * A map of keywords from the parsed reply. + * For example, if the reply is "A=X B C=YZ", then the map will be + * keys["A"] == "X" + * keys["B"] == (empty std::optional) + * keys["C"] == "YZ" + */ + std::unordered_map<std::string, std::optional<std::string>> keys; + + /** + * Get the value of a given key. + * For example if the reply is "A=X B" then: + * Value("A") -> "X" + * Value("B") -> throws + * Value("C") -> throws + * @param[in] key Key whose value to retrieve + * @returns the key's value + * @throws std::runtime_error if the key is not present or if it has no value + */ + std::string Get(const std::string& key) const; + }; + + /** + * Log a message in the `BCLog::I2P` category. + * @param[in] fmt printf(3)-like format string. + * @param[in] args printf(3)-like arguments that correspond to `fmt`. + */ + template <typename... Args> + void Log(const std::string& fmt, const Args&... args) const; + + /** + * Send request and get a reply from the SAM proxy. + * @param[in] sock A socket that is connected to the SAM proxy. + * @param[in] request Raw request to send, a newline terminator is appended to it. + * @param[in] check_result_ok If true then after receiving the reply a check is made + * whether it contains "RESULT=OK" and an exception is thrown if it does not. + * @throws std::runtime_error if an error occurs + */ + Reply SendRequestAndGetReply(const Sock& sock, + const std::string& request, + bool check_result_ok = true) const; + + /** + * Open a new connection to the SAM proxy. + * @return a connected socket + * @throws std::runtime_error if an error occurs + */ + Sock Hello() const EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Check the control socket for errors and possibly disconnect. + */ + void CheckControlSock(); + + /** + * Generate a new destination with the SAM proxy and set `m_private_key` to it. + * @param[in] sock Socket to use for talking to the SAM proxy. + * @throws std::runtime_error if an error occurs + */ + void DestGenerate(const Sock& sock) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Generate a new destination with the SAM proxy, set `m_private_key` to it and save + * it on disk to `m_private_key_file`. + * @param[in] sock Socket to use for talking to the SAM proxy. + * @throws std::runtime_error if an error occurs + */ + void GenerateAndSavePrivateKey(const Sock& sock) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Derive own destination from `m_private_key`. + * @see https://geti2p.net/spec/common-structures#destination + * @return an I2P destination + */ + Binary MyDestination() const EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Create the session if not already created. Reads the private key file and connects to the + * SAM proxy. + * @throws std::runtime_error if an error occurs + */ + void CreateIfNotCreatedAlready() EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Open a new connection to the SAM proxy and issue "STREAM ACCEPT" request using the existing + * session id. Return the idle socket that is waiting for a peer to connect to us. + * @throws std::runtime_error if an error occurs + */ + Sock StreamAccept() EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * Destroy the session, closing the internally used sockets. + */ + void Disconnect() EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + + /** + * The name of the file where this peer's private key is stored (in binary). + */ + const fs::path m_private_key_file; + + /** + * The host and port of the SAM control service. + */ + const CService m_control_host; + + /** + * Cease network activity when this is signaled. + */ + CThreadInterrupt* const m_interrupt; + + /** + * Mutex protecting the members that can be concurrently accessed. + */ + mutable Mutex m_mutex; + + /** + * The private key of this peer. + * @see The reply to the "DEST GENERATE" command in https://geti2p.net/en/docs/api/samv3 + */ + Binary m_private_key GUARDED_BY(m_mutex); + + /** + * SAM control socket. + * Used to connect to the I2P SAM service and create a session + * ("SESSION CREATE"). With the established session id we later open + * other connections to the SAM service to accept incoming I2P + * connections and make outgoing ones. + * See https://geti2p.net/en/docs/api/samv3 + */ + Sock m_control_sock GUARDED_BY(m_mutex); + + /** + * Our .b32.i2p address. + * Derived from `m_private_key`. + */ + CService m_my_addr GUARDED_BY(m_mutex); + + /** + * SAM session id. + */ + std::string m_session_id GUARDED_BY(m_mutex); +}; + +} // namespace sam +} // namespace i2p + +#endif /* BITCOIN_I2P_H */ diff --git a/src/index/base.cpp b/src/index/base.cpp index e67b813763..25644c3b41 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -62,9 +62,46 @@ bool BaseIndex::Init() if (locator.IsNull()) { m_best_block_index = nullptr; } else { - m_best_block_index = FindForkInGlobalIndex(::ChainActive(), locator); + m_best_block_index = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); } m_synced = m_best_block_index.load() == ::ChainActive().Tip(); + if (!m_synced) { + bool prune_violation = false; + if (!m_best_block_index) { + // index is not built yet + // make sure we have all block data back to the genesis + const CBlockIndex* block = ::ChainActive().Tip(); + while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { + block = block->pprev; + } + prune_violation = block != ::ChainActive().Genesis(); + } + // in case the index has a best block set and is not fully synced + // check if we have the required blocks to continue building the index + else { + const CBlockIndex* block_to_test = m_best_block_index.load(); + if (!ChainActive().Contains(block_to_test)) { + // if the bestblock is not part of the mainchain, find the fork + // and make sure we have all data down to the fork + block_to_test = ::ChainActive().FindFork(block_to_test); + } + const CBlockIndex* block = ::ChainActive().Tip(); + prune_violation = true; + // check backwards from the tip if we have all block data until we reach the indexes bestblock + while (block_to_test && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { + if (block_to_test == block) { + prune_violation = false; + break; + } + block = block->pprev; + } + } + if (prune_violation) { + // throw error and graceful shutdown if we can't build the index + FatalError("%s: %s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", __func__, GetName()); + return false; + } + } return true; } @@ -177,6 +214,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); // In the case of a reorg, ensure persisted block locator is not stale. + // Pruning has a minimum of 288 blocks-to-keep and getting the index + // out of sync may be possible but a users fault. + // In case we reorg beyond the pruned depth, ReadBlockFromDisk would + // throw and lead to a graceful shutdown m_best_block_index = new_tip; if (!Commit()) { // If commit fails, revert the best block index to avoid corruption. @@ -239,7 +280,7 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) const CBlockIndex* locator_tip_index; { LOCK(cs_main); - locator_tip_index = LookupBlockIndex(locator_tip_hash); + locator_tip_index = g_chainman.m_blockman.LookupBlockIndex(locator_tip_hash); } if (!locator_tip_index) { @@ -325,6 +366,6 @@ IndexSummary BaseIndex::GetSummary() const IndexSummary summary{}; summary.name = GetName(); summary.synced = m_synced; - summary.best_block_height = m_best_block_index.load()->nHeight; + summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0; return summary; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index a9c8188bb6..221ac02c9e 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -9,15 +9,11 @@ #include <chain.h> #include <flatfile.h> #include <index/base.h> +#include <util/hasher.h> /** Interval between compact filter checkpoints. See BIP 157. */ static constexpr int CFCHECKPT_INTERVAL = 1000; -struct FilterHeaderHasher -{ - size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } -}; - /** * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of * blocks by height. An index is constructed for each supported filter type with its own database diff --git a/src/init.cpp b/src/init.cpp index 9a63fe0e03..f40cf7975b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -68,6 +68,8 @@ #include <set> #include <stdint.h> #include <stdio.h> +#include <thread> +#include <vector> #ifndef WIN32 #include <attributes.h> @@ -78,7 +80,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/signals2/signal.hpp> -#include <boost/thread/thread.hpp> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -155,8 +156,6 @@ static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; static std::thread g_load_block; -static boost::thread_group threadGroup; - void Interrupt(NodeContext& node) { InterruptHTTPServer(); @@ -218,11 +217,10 @@ void Shutdown(NodeContext& node) StopTorControl(); // After everything has been shut down, but before things get flushed, stop the - // CScheduler/checkqueue, threadGroup and load block thread. + // CScheduler/checkqueue, scheduler and load block thread. if (node.scheduler) node.scheduler->stop(); if (g_load_block.joinable()) g_load_block.join(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. @@ -388,6 +386,7 @@ void SetupServerArgs(NodeContext& node) #endif argsman.AddArg("-assumevalid=<hex>", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), signetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksdir=<dir>", "Specify directory to hold blocks subdirectory for *.dat files (default: <datadir>)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); #if HAVE_SYSTEM argsman.AddArg("-blocknotify=<cmd>", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif @@ -436,8 +435,9 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-dns", strprintf("Allow DNS lookups for -addnode, -seednode and -connect (default: %u)", DEFAULT_NAME_LOOKUP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-dnsseed", "Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-dnsseed", strprintf("Query for peer addresses via DNS lookup, if low on addresses (default: %u unless -connect used)", DEFAULT_DNSSEED), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); argsman.AddArg("-externalip=<ip>", "Specify your own public address", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-fixedseeds", strprintf("Allow fixed seeds if DNS seeds don't provide peers (default: %u)", DEFAULT_FIXEDSEEDS), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", "Accept connections from outside (default: 1 if no -proxy or -connect)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -447,7 +447,9 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); + argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (" + Join(GetNetworkNames(), ", ") + "). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks. Warning: if it is used with non-onion networks and the -onion or -proxy option is set, then outbound onion connections will still be made; use -noonion or -onion=0 to disable outbound onion connections in this case.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -456,8 +458,8 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP @@ -521,10 +523,11 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); #ifdef HAVE_THREAD_LOCAL @@ -532,6 +535,7 @@ void SetupServerArgs(NodeContext& node) #else hidden_args.emplace_back("-logthreadnames"); #endif + argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -704,7 +708,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp if (!file) break; // This error is logged in OpenBlockFile LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - LoadExternalBlockFile(chainparams, file, &pos); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file, &pos); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -723,7 +727,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); - LoadExternalBlockFile(chainparams, file); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -773,6 +777,10 @@ static bool InitSanityCheck() return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); } + if (!ChronoSanityCheck()) { + return InitError(Untranslated("Clock epoch mismatch. Aborting.")); + } + return true; } @@ -841,6 +849,9 @@ void InitParameterInteraction(ArgsManager& args) LogPrintf("%s: parameter interaction: -listen=0 -> setting -discover=0\n", __func__); if (args.SoftSetBoolArg("-listenonion", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -listenonion=0\n", __func__); + if (args.SoftSetBoolArg("-i2pacceptincoming", false)) { + LogPrintf("%s: parameter interaction: -listen=0 -> setting -i2pacceptincoming=0\n", __func__); + } } if (args.IsArgSet("-externalip")) { @@ -878,6 +889,7 @@ void InitLogging(const ArgsManager& args) #ifdef HAVE_THREAD_LOCAL LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); #endif + LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); @@ -1024,9 +1036,6 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.GetArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); - if (!g_enabled_filter_types.empty()) { - return InitError(_("Prune mode is incompatible with -blockfilterindex.")); - } } // -bind and -whitebind can't be set when not listening @@ -1042,16 +1051,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Trim requested connection counts, to fit into system limitations // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 - nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind); + nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); + #ifdef USE_POLL int fd_max = nFD; #else int fd_max = FD_SETSIZE; #endif - nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); + nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); - nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); + nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE, nMaxConnections); if (nMaxConnections < nUserMaxConnections) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); @@ -1334,16 +1344,14 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA LogPrintf("Script verification uses %d additional threads\n", script_threads); if (script_threads >= 1) { g_parallel_script_checks = true; - for (int i = 0; i < script_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_threads); } assert(!node.scheduler); node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); + node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1410,8 +1418,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA ChainstateManager& chainman = *Assert(node.chainman); assert(!node.peerman); - node.peerman = std::make_unique<PeerManager>(chainparams, *node.connman, node.banman.get(), - *node.scheduler, chainman, *node.mempool, ignores_incoming_txs); + node.peerman = PeerManager::make(chainparams, *node.connman, node.banman.get(), + *node.scheduler, chainman, *node.mempool, ignores_incoming_txs); RegisterValidationInterface(node.peerman.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1612,7 +1620,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). if (!chainman.BlockIndex().empty() && - !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { + !g_chainman.m_blockman.LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } @@ -1987,6 +1995,21 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA connOptions.m_specified_outgoing = connect; } } + + const std::string& i2psam_arg = args.GetArg("-i2psam", ""); + if (!i2psam_arg.empty()) { + CService addr; + if (!Lookup(i2psam_arg, addr, 7656, fNameLookup) || !addr.IsValid()) { + return InitError(strprintf(_("Invalid -i2psam address or hostname: '%s'"), i2psam_arg)); + } + SetReachable(NET_I2P, true); + SetProxy(NET_I2P, proxyType{addr}); + } else { + SetReachable(NET_I2P, false); + } + + connOptions.m_i2p_accept_incoming = args.GetBoolArg("-i2pacceptincoming", true); + if (!node.connman->Start(*node.scheduler, connOptions)) { return false; } diff --git a/src/key_io.cpp b/src/key_io.cpp index d2f5be93f5..e27673fd16 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -8,16 +8,15 @@ #include <bech32.h> #include <util/strencodings.h> -#include <boost/variant/apply_visitor.hpp> -#include <boost/variant/static_visitor.hpp> - +#include <algorithm> #include <assert.h> #include <string.h> -#include <algorithm> -namespace -{ -class DestinationEncoder : public boost::static_visitor<std::string> +/// Maximum witness length for Bech32 addresses. +static constexpr std::size_t BECH32_WITNESS_PROG_MAX_LEN = 40; + +namespace { +class DestinationEncoder { private: const CChainParams& m_params; @@ -69,10 +68,11 @@ public: std::string operator()(const CNoDestination& no) const { return {}; } }; -CTxDestination DecodeDestination(const std::string& str, const CChainParams& params) +CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str) { std::vector<unsigned char> data; uint160 hash; + error_str = ""; if (DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). @@ -89,10 +89,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin() + script_prefix.size(), data.end(), hash.begin()); return ScriptHash(hash); } + + // Set potential error message. + // This message may be changed if the address can also be interpreted as a Bech32 address. + error_str = "Invalid prefix for Base58-encoded address"; } data.clear(); auto bech = bech32::Decode(str); - if (bech.second.size() > 0 && bech.first == params.Bech32HRP()) { + if (bech.second.size() > 0) { + error_str = ""; + + if (bech.first != params.Bech32HRP()) { + error_str = "Invalid prefix for Bech32 address"; + return CNoDestination(); + } + // Bech32 decoding int version = bech.second[0]; // The first 5 bit symbol is the witness version (0-16) // The rest of the symbols are converted witness program bytes. @@ -113,11 +124,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return scriptid; } } + + error_str = "Invalid Bech32 v0 address data size"; return CNoDestination(); } - if (version > 16 || data.size() < 2 || data.size() > 40) { + + if (version > 16) { + error_str = "Invalid Bech32 address witness version"; return CNoDestination(); } + + if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { + error_str = "Invalid Bech32 address data size"; + return CNoDestination(); + } + WitnessUnknown unk; unk.version = version; std::copy(data.begin(), data.end(), unk.program); @@ -125,6 +146,10 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return unk; } } + + // Set error message if address can't be interpreted as Base58 or Bech32. + if (error_str.empty()) error_str = "Invalid address format"; + return CNoDestination(); } } // namespace @@ -209,17 +234,24 @@ std::string EncodeExtKey(const CExtKey& key) std::string EncodeDestination(const CTxDestination& dest) { - return boost::apply_visitor(DestinationEncoder(Params()), dest); + return std::visit(DestinationEncoder(Params()), dest); +} + +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg) +{ + return DecodeDestination(str, Params(), error_msg); } CTxDestination DecodeDestination(const std::string& str) { - return DecodeDestination(str, Params()); + std::string error_msg; + return DecodeDestination(str, error_msg); } bool IsValidDestinationString(const std::string& str, const CChainParams& params) { - return IsValidDestination(DecodeDestination(str, params)); + std::string error_msg; + return IsValidDestination(DecodeDestination(str, params, error_msg)); } bool IsValidDestinationString(const std::string& str) diff --git a/src/key_io.h b/src/key_io.h index d80c08f49c..bd81f7847e 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -23,6 +23,7 @@ std::string EncodeExtPubKey(const CExtPubKey& extpubkey); std::string EncodeDestination(const CTxDestination& dest); CTxDestination DecodeDestination(const std::string& str); +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg); bool IsValidDestinationString(const std::string& str); bool IsValidDestinationString(const std::string& str, const CChainParams& params); diff --git a/src/logging.cpp b/src/logging.cpp index 4ddcf1d930..866213786e 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -5,6 +5,7 @@ #include <logging.h> #include <util/threadnames.h> +#include <util/string.h> #include <util/time.h> #include <mutex> @@ -155,6 +156,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, + {BCLog::I2P, "i2p"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -203,9 +205,9 @@ std::string BCLog::Logger::LogTimestampStr(const std::string& str) strStamped.pop_back(); strStamped += strprintf(".%06dZ", nTimeMicros%1000000); } - int64_t mocktime = GetMockTime(); - if (mocktime) { - strStamped += " (mocktime: " + FormatISO8601DateTime(mocktime) + ")"; + std::chrono::seconds mocktime = GetMockTime(); + if (mocktime > 0s) { + strStamped += " (mocktime: " + FormatISO8601DateTime(count_seconds(mocktime)) + ")"; } strStamped += ' ' + str; } else @@ -236,11 +238,15 @@ namespace BCLog { } } -void BCLog::Logger::LogPrintStr(const std::string& str) +void BCLog::Logger::LogPrintStr(const std::string& str, const std::string& logging_function, const std::string& source_file, const int source_line) { StdLockGuard scoped_lock(m_cs); std::string str_prefixed = LogEscapeMessage(str); + if (m_log_sourcelocations && m_started_new_line) { + str_prefixed.insert(0, "[" + RemovePrefix(source_file, "./") + ":" + ToString(source_line) + "] [" + logging_function + "] "); + } + if (m_log_threadnames && m_started_new_line) { str_prefixed.insert(0, "[" + util::ThreadGetInternalName() + "] "); } diff --git a/src/logging.h b/src/logging.h index 9efecc7c12..436f0cd12e 100644 --- a/src/logging.h +++ b/src/logging.h @@ -22,6 +22,7 @@ static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; static const bool DEFAULT_LOGTHREADNAMES = false; +static const bool DEFAULT_LOGSOURCELOCATIONS = false; extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; @@ -56,6 +57,7 @@ namespace BCLog { QT = (1 << 19), LEVELDB = (1 << 20), VALIDATION = (1 << 21), + I2P = (1 << 22), ALL = ~(uint32_t)0, }; @@ -90,12 +92,13 @@ namespace BCLog { bool m_log_timestamps = DEFAULT_LOGTIMESTAMPS; bool m_log_time_micros = DEFAULT_LOGTIMEMICROS; bool m_log_threadnames = DEFAULT_LOGTHREADNAMES; + bool m_log_sourcelocations = DEFAULT_LOGSOURCELOCATIONS; fs::path m_file_path; std::atomic<bool> m_reopen_file{false}; /** Send a string to the log output */ - void LogPrintStr(const std::string& str); + void LogPrintStr(const std::string& str, const std::string& logging_function, const std::string& source_file, const int source_line); /** Returns whether logs will be written to any output */ bool Enabled() const @@ -163,7 +166,7 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); // peer can fill up a user's disk with debug.log entries. template <typename... Args> -static inline void LogPrintf(const char* fmt, const Args&... args) +static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const char* fmt, const Args&... args) { if (LogInstance().Enabled()) { std::string log_msg; @@ -173,10 +176,12 @@ static inline void LogPrintf(const char* fmt, const Args&... args) /* Original format string will have newline so don't add one here */ log_msg = "Error \"" + std::string(fmterr.what()) + "\" while formatting log message: " + fmt; } - LogInstance().LogPrintStr(log_msg); + LogInstance().LogPrintStr(log_msg, logging_function, source_file, source_line); } } +#define LogPrintf(...) LogPrintf_(__func__, __FILE__, __LINE__, __VA_ARGS__) + // Use a macro instead of a function for conditional logging to prevent // evaluating arguments when logging for the category is not enabled. #define LogPrint(category, ...) \ diff --git a/src/miner.cpp b/src/miner.cpp index 009ae6b13a..076d43c951 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,7 @@ void RegenerateCommitments(CBlock& block) tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); block.vtx.at(0) = MakeTransactionRef(tx); - GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); block.hashMerkleRoot = BlockMerkleRoot(block); } @@ -176,7 +176,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } int64_t nTime2 = GetTimeMicros(); diff --git a/src/net.cpp b/src/net.cpp index 6e5a271945..6dcff7e32e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -11,8 +11,10 @@ #include <banman.h> #include <clientversion.h> +#include <compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> +#include <i2p.h> #include <net_permissions.h> #include <netbase.h> #include <node/ui_interface.h> @@ -20,6 +22,7 @@ #include <protocol.h> #include <random.h> #include <scheduler.h> +#include <util/sock.h> #include <util/strencodings.h> #include <util/translation.h> @@ -35,6 +38,7 @@ #include <algorithm> #include <cstdint> +#include <functional> #include <unordered_map> #include <math.h> @@ -70,16 +74,6 @@ static constexpr std::chrono::seconds MAX_UPLOAD_TIMEFRAME{60 * 60 * 24}; // We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. #define FEELER_SLEEP_WINDOW 1 -// MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 -#if !defined(MSG_NOSIGNAL) -#define MSG_NOSIGNAL 0 -#endif - -// MSG_DONTWAIT is not available on some platforms, if it doesn't exist define it as 0 -#if !defined(MSG_DONTWAIT) -#define MSG_DONTWAIT 0 -#endif - /** Used to pass flags to the Bind() function */ enum BindFlags { BF_NONE = 0, @@ -199,31 +193,29 @@ bool IsPeerAddrLocalGood(CNode *pnode) IsReachable(addrLocal.GetNetwork()); } -// pushes our own address to a peer -void AdvertiseLocal(CNode *pnode) +Optional<CAddress> GetLocalAddrForPeer(CNode *pnode) { - if (fListen && pnode->fSuccessfullyConnected) + CAddress addrLocal = GetLocalAddress(&pnode->addr, pnode->GetLocalServices()); + if (gArgs.GetBoolArg("-addrmantest", false)) { + // use IPv4 loopback during addrmantest + addrLocal = CAddress(CService(LookupNumeric("127.0.0.1", GetListenPort())), pnode->GetLocalServices()); + } + // If discovery is enabled, sometimes give our peer the address it + // tells us that it sees us as in case it has a better idea of our + // address than we do. + FastRandomContext rng; + if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || + rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { - CAddress addrLocal = GetLocalAddress(&pnode->addr, pnode->GetLocalServices()); - if (gArgs.GetBoolArg("-addrmantest", false)) { - // use IPv4 loopback during addrmantest - addrLocal = CAddress(CService(LookupNumeric("127.0.0.1", GetListenPort())), pnode->GetLocalServices()); - } - // If discovery is enabled, sometimes give our peer the address it - // tells us that it sees us as in case it has a better idea of our - // address than we do. - FastRandomContext rng; - if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || - rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) - { - addrLocal.SetIP(pnode->GetAddrLocal()); - } - if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) - { - LogPrint(BCLog::NET, "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); - pnode->PushAddress(addrLocal, rng); - } + addrLocal.SetIP(pnode->GetAddrLocal()); + } + if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) + { + LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToString(), pnode->GetId()); + return addrLocal; } + // Address is unroutable. Don't advertise. + return nullopt; } // learn a new local address @@ -428,24 +420,36 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Connect bool connected = false; - SOCKET hSocket = INVALID_SOCKET; + std::unique_ptr<Sock> sock; proxyType proxy; + CAddress addr_bind; + assert(!addr_bind.IsValid()); + if (addrConnect.IsValid()) { bool proxyConnectionFailed = false; - if (GetProxy(addrConnect.GetNetwork(), proxy)) { - hSocket = CreateSocket(proxy.proxy); - if (hSocket == INVALID_SOCKET) { + if (addrConnect.GetNetwork() == NET_I2P && m_i2p_sam_session.get() != nullptr) { + i2p::Connection conn; + if (m_i2p_sam_session->Connect(addrConnect, conn, proxyConnectionFailed)) { + connected = true; + sock = std::make_unique<Sock>(std::move(conn.sock)); + addr_bind = CAddress{conn.me, NODE_NONE}; + } + } else if (GetProxy(addrConnect.GetNetwork(), proxy)) { + sock = CreateSock(proxy.proxy); + if (!sock) { return nullptr; } - connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), + *sock, nConnectTimeout, proxyConnectionFailed); } else { // no proxy needed (none set for target network) - hSocket = CreateSocket(addrConnect); - if (hSocket == INVALID_SOCKET) { + sock = CreateSock(addrConnect); + if (!sock) { return nullptr; } - connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout, conn_type == ConnectionType::MANUAL); + connected = ConnectSocketDirectly(addrConnect, sock->Get(), nConnectTimeout, + conn_type == ConnectionType::MANUAL); } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to @@ -453,26 +457,28 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addrman.Attempt(addrConnect, fCountFailure); } } else if (pszDest && GetNameProxy(proxy)) { - hSocket = CreateSocket(proxy.proxy); - if (hSocket == INVALID_SOCKET) { + sock = CreateSock(proxy.proxy); + if (!sock) { return nullptr; } std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); bool proxyConnectionFailed; - connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, host, port, *sock, nConnectTimeout, + proxyConnectionFailed); } if (!connected) { - CloseSocket(hSocket); return nullptr; } // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); + if (!addr_bind.IsValid()) { + addr_bind = GetBindAddress(sock->Get()); + } + CNode* pnode = new CNode(id, nLocalServices, sock->Release(), addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type, /* inbound_onion */ false); pnode->AddRef(); // We're making a new connection, harvest entropy from the time (and our peer count) @@ -498,9 +504,9 @@ void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNet } } -std::string CNode::ConnectionTypeAsString() const +std::string ConnectionTypeAsString(ConnectionType conn_type) { - switch (m_conn_type) { + switch (conn_type) { case ConnectionType::INBOUND: return "inbound"; case ConnectionType::MANUAL: @@ -592,33 +598,19 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) } X(m_permissionFlags); if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_feeFilter); stats.minFeeFilter = m_tx_relay->minFeeFilter; } else { stats.minFeeFilter = 0; } - // It is common for nodes with good ping times to suddenly become lagged, - // due to a new block arriving or other large transfer. - // Merely reporting pingtime might fool the caller into thinking the node was still responsive, - // since pingtime does not update until the ping is complete, which might take a while. - // So, if a ping is taking an unusually long time in flight, - // the caller can immediately detect that this is happening. - std::chrono::microseconds ping_wait{0}; - if ((0 != nPingNonceSent) && (0 != m_ping_start.load().count())) { - ping_wait = GetTime<std::chrono::microseconds>() - m_ping_start.load(); - } - - // Raw ping time is in microseconds, but show it to user as whole seconds (Bitcoin users should be well used to small numbers with many decimal places by now :) - stats.m_ping_usec = nPingUsecTime; - stats.m_min_ping_usec = nMinPingUsecTime; - stats.m_ping_wait_usec = count_microseconds(ping_wait); + stats.m_ping_usec = m_last_ping_time; + stats.m_min_ping_usec = m_min_ping_time; // Leave string empty if addrLocal invalid (not filled in yet) CService addrLocalUnlocked = GetAddrLocal(); stats.addrLocal = addrLocalUnlocked.IsValid() ? addrLocalUnlocked.ToString() : ""; - stats.m_conn_type_string = ConnectionTypeAsString(); + X(m_conn_type); } #undef X @@ -815,7 +807,7 @@ size_t CConnman::SocketSendData(CNode& node) const // error int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { - LogPrintf("socket send error %s\n", NetworkErrorString(nErr)); + LogPrint(BCLog::NET, "socket send error for peer=%d: %s\n", node.GetId(), NetworkErrorString(nErr)); node.CloseSocketDisconnect(); } } @@ -834,7 +826,7 @@ size_t CConnman::SocketSendData(CNode& node) const static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a.nMinPingUsecTime > b.nMinPingUsecTime; + return a.m_min_ping_time > b.m_min_ping_time; } static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) @@ -989,7 +981,7 @@ bool CConnman::AttemptToEvictConnection() peer_relay_txes = node->m_tx_relay->fRelayTxes; peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; } - NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->m_min_ping_time, node->nLastBlockTime, node->nLastTXTime, HasAllDesirableServiceFlags(node->nServices), peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, @@ -1004,6 +996,7 @@ bool CConnman::AttemptToEvictConnection() LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (pnode->GetId() == *node_id_to_evict) { + LogPrint(BCLog::NET, "selected %s connection for eviction peer=%d; disconnecting\n", pnode->ConnectionTypeAsString(), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -1016,17 +1009,35 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { socklen_t len = sizeof(sockaddr); SOCKET hSocket = accept(hListenSocket.socket, (struct sockaddr*)&sockaddr, &len); CAddress addr; - int nInbound = 0; - int nMaxInbound = nMaxConnections - m_max_outbound; - if (hSocket != INVALID_SOCKET) { - if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { - LogPrintf("Warning: Unknown socket family\n"); + if (hSocket == INVALID_SOCKET) { + const int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK) { + LogPrintf("socket error accept failed: %s\n", NetworkErrorString(nErr)); } + return; } + if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { + LogPrintf("Warning: Unknown socket family\n"); + } + + const CAddress addr_bind = GetBindAddress(hSocket); + NetPermissionFlags permissionFlags = NetPermissionFlags::PF_NONE; hListenSocket.AddSocketPermissionFlags(permissionFlags); + + CreateNodeFromAcceptedSocket(hSocket, permissionFlags, addr_bind, addr); +} + +void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, + NetPermissionFlags permissionFlags, + const CAddress& addr_bind, + const CAddress& addr) +{ + int nInbound = 0; + int nMaxInbound = nMaxConnections - m_max_outbound; + AddWhitelistPermissionFlags(permissionFlags, addr); if (NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_ISIMPLICIT)) { NetPermissions::ClearFlag(permissionFlags, PF_ISIMPLICIT); @@ -1043,16 +1054,8 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } } - if (hSocket == INVALID_SOCKET) - { - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK) - LogPrintf("socket error accept failed: %s\n", NetworkErrorString(nErr)); - return; - } - if (!fNetworkActive) { - LogPrintf("connection from %s dropped: not accepting new connections\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s dropped: not accepting new connections\n", addr.ToString()); CloseSocket(hSocket); return; } @@ -1098,7 +1101,6 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CAddress addr_bind = GetBindAddress(hSocket); ServiceFlags nodeServices = nLocalServices; if (NetPermissions::HasFlag(permissionFlags, PF_BLOOMFILTER)) { @@ -1123,6 +1125,27 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { RandAddEvent((uint32_t)id); } +bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type) +{ + if (conn_type != ConnectionType::OUTBOUND_FULL_RELAY && conn_type != ConnectionType::BLOCK_RELAY) return false; + + const int max_connections = conn_type == ConnectionType::OUTBOUND_FULL_RELAY ? m_max_outbound_full_relay : m_max_outbound_block_relay; + + // Count existing connections + int existing_connections = WITH_LOCK(cs_vNodes, + return std::count_if(vNodes.begin(), vNodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); + + // Max connections of specified type already exist + if (existing_connections >= max_connections) return false; + + // Max total outbound connections already exist + CSemaphoreGrant grant(*semOutbound, true); + if (!grant) return false; + + OpenNetworkConnection(CAddress(), false, &grant, address.c_str(), conn_type); + return true; +} + void CConnman::DisconnectNodes() { { @@ -1196,37 +1219,38 @@ void CConnman::NotifyNumConnectionsChanged() } } -void CConnman::InactivityCheck(CNode *pnode) const +bool CConnman::RunInactivityChecks(const CNode& node) const { - int64_t nTime = GetSystemTimeInSeconds(); - if (nTime - pnode->nTimeConnected > m_peer_connect_timeout) - { - if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) - { - LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d from %d\n", m_peer_connect_timeout, pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); - pnode->fDisconnect = true; - } - else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) - { - LogPrintf("socket sending timeout: %is\n", nTime - pnode->nLastSend); - pnode->fDisconnect = true; - } - else if (nTime - pnode->nLastRecv > TIMEOUT_INTERVAL) - { - LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); - pnode->fDisconnect = true; - } - else if (pnode->nPingNonceSent && pnode->m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) - { - LogPrintf("ping timeout: %fs\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - pnode->m_ping_start.load())); - pnode->fDisconnect = true; - } - else if (!pnode->fSuccessfullyConnected) - { - LogPrint(BCLog::NET, "version handshake timeout from %d\n", pnode->GetId()); - pnode->fDisconnect = true; - } + return GetSystemTimeInSeconds() > node.nTimeConnected + m_peer_connect_timeout; +} + +bool CConnman::InactivityCheck(const CNode& node) const +{ + // Use non-mockable system time (otherwise these timers will pop when we + // use setmocktime in the tests). + int64_t now = GetSystemTimeInSeconds(); + + if (node.nLastRecv == 0 || node.nLastSend == 0) { + LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); + return true; } + + if (now > node.nLastSend + TIMEOUT_INTERVAL) { + LogPrint(BCLog::NET, "socket sending timeout: %is peer=%d\n", now - node.nLastSend, node.GetId()); + return true; + } + + if (now > node.nLastRecv + TIMEOUT_INTERVAL) { + LogPrint(BCLog::NET, "socket receive timeout: %is peer=%d\n", now - node.nLastRecv, node.GetId()); + return true; + } + + if (!node.fSuccessfullyConnected) { + LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId()); + return true; + } + + return false; } bool CConnman::GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set) @@ -1502,7 +1526,7 @@ void CConnman::SocketHandler() if (bytes_sent) RecordBytesSent(bytes_sent); } - InactivityCheck(pnode); + if (RunInactivityChecks(*pnode) && InactivityCheck(*pnode)) pnode->fDisconnect = true; } { LOCK(cs_vNodes); @@ -1737,11 +1761,19 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Initiate network connections - int64_t nStart = GetTime(); + auto start = GetTime<std::chrono::seconds>(); // Minimum time before next feeler connection (in microseconds). - int64_t nNextFeeler = PoissonNextSend(nStart*1000*1000, FEELER_INTERVAL); - int64_t nNextExtraBlockRelay = PoissonNextSend(nStart*1000*1000, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + + int64_t nNextFeeler = PoissonNextSend(count_microseconds(start), FEELER_INTERVAL); + int64_t nNextExtraBlockRelay = PoissonNextSend(count_microseconds(start), EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + const bool dnsseed = gArgs.GetBoolArg("-dnsseed", DEFAULT_DNSSEED); + bool add_fixed_seeds = gArgs.GetBoolArg("-fixedseeds", DEFAULT_FIXEDSEEDS); + + if (!add_fixed_seeds) { + LogPrintf("Fixed seeds are disabled\n"); + } + while (!interruptNet) { ProcessAddrFetch(); @@ -1753,18 +1785,32 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (interruptNet) return; - // Add seed nodes if DNS seeds are all down (an infrastructure attack?). - // Note that we only do this if we started with an empty peers.dat, - // (in which case we will query DNS seeds immediately) *and* the DNS - // seeds have not returned any results. - if (addrman.size() == 0 && (GetTime() - nStart > 60)) { - static bool done = false; - if (!done) { - LogPrintf("Adding fixed seed nodes as DNS doesn't seem to be available.\n"); + if (add_fixed_seeds && addrman.size() == 0) { + // When the node starts with an empty peers.dat, there are a few other sources of peers before + // we fallback on to fixed seeds: -dnsseed, -seednode, -addnode + // If none of those are available, we fallback on to fixed seeds immediately, else we allow + // 60 seconds for any of those sources to populate addrman. + bool add_fixed_seeds_now = false; + // It is cheapest to check if enough time has passed first. + if (GetTime<std::chrono::seconds>() > start + std::chrono::minutes{1}) { + add_fixed_seeds_now = true; + LogPrintf("Adding fixed seeds as 60 seconds have passed and addrman is empty\n"); + } + + // Checking !dnsseed is cheaper before locking 2 mutexes. + if (!add_fixed_seeds_now && !dnsseed) { + LOCK2(m_addr_fetches_mutex, cs_vAddedNodes); + if (m_addr_fetches.empty() && vAddedNodes.empty()) { + add_fixed_seeds_now = true; + LogPrintf("Adding fixed seeds as -dnsseed=0, -addnode is not provided and all -seednode(s) attempted\n"); + } + } + + if (add_fixed_seeds_now) { CNetAddr local; local.SetInternal("fixedseeds"); addrman.Add(convertSeed6(Params().FixedSeeds()), local); - done = true; + add_fixed_seeds = false; } } @@ -2142,6 +2188,45 @@ void CConnman::ThreadMessageHandler() } } +void CConnman::ThreadI2PAcceptIncoming() +{ + static constexpr auto err_wait_begin = 1s; + static constexpr auto err_wait_cap = 5min; + auto err_wait = err_wait_begin; + + bool advertising_listen_addr = false; + i2p::Connection conn; + + while (!interruptNet) { + + if (!m_i2p_sam_session->Listen(conn)) { + if (advertising_listen_addr && conn.me.IsValid()) { + RemoveLocal(conn.me); + advertising_listen_addr = false; + } + + interruptNet.sleep_for(err_wait); + if (err_wait < err_wait_cap) { + err_wait *= 2; + } + + continue; + } + + if (!advertising_listen_addr) { + AddLocal(conn.me, LOCAL_BIND); + advertising_listen_addr = true; + } + + if (!m_i2p_sam_session->Accept(conn)) { + continue; + } + + CreateNodeFromAcceptedSocket(conn.sock.Release(), NetPermissionFlags::PF_NONE, + CAddress{conn.me, NODE_NONE}, CAddress{conn.peer, NODE_NONE}); + } +} + bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, NetPermissionFlags permissions) { int nOne = 1; @@ -2156,9 +2241,8 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, return false; } - SOCKET hListenSocket = CreateSocket(addrBind); - if (hListenSocket == INVALID_SOCKET) - { + std::unique_ptr<Sock> sock = CreateSock(addrBind); + if (!sock) { strError = strprintf(Untranslated("Error: Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError.original); return false; @@ -2166,21 +2250,21 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. - setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)); + setsockopt(sock->Get(), SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)); // some systems don't have IPV6_V6ONLY but are always v6only; others do have the option // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { #ifdef IPV6_V6ONLY - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); + setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); #endif #ifdef WIN32 int nProtLevel = PROTECTION_LEVEL_UNRESTRICTED; - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)); + setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)); #endif } - if (::bind(hListenSocket, (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) + if (::bind(sock->Get(), (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) @@ -2188,21 +2272,19 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, else strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)"), addrBind.ToString(), NetworkErrorString(nErr)); LogPrintf("%s\n", strError.original); - CloseSocket(hListenSocket); return false; } LogPrintf("Bound to %s\n", addrBind.ToString()); // Listen for incoming connections - if (listen(hListenSocket, SOMAXCONN) == SOCKET_ERROR) + if (listen(sock->Get(), SOMAXCONN) == SOCKET_ERROR) { strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError.original); - CloseSocket(hListenSocket); return false; } - vhListenSocket.push_back(ListenSocket(hListenSocket, permissions)); + vhListenSocket.push_back(ListenSocket(sock->Release(), permissions)); return true; } @@ -2344,6 +2426,12 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) return false; } + proxyType i2p_sam; + if (GetProxy(NET_I2P, i2p_sam)) { + m_i2p_sam_session = std::make_unique<i2p::sam::Session>(GetDataDir() / "i2p_private_key", + i2p_sam.proxy, &interruptNet); + } + for (const auto& strDest : connOptions.vSeedNodes) { AddAddrFetch(strDest); } @@ -2359,7 +2447,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); else { addrman.Clear(); // Addrman can be in an inconsistent state after failure, reset it - LogPrintf("Invalid or missing peers.dat; recreating\n"); + LogPrintf("Recreating peers.dat\n"); DumpAddresses(); } } @@ -2402,7 +2490,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) // Send and receive from sockets, accept connections threadSocketHandler = std::thread(&TraceThread<std::function<void()> >, "net", std::function<void()>(std::bind(&CConnman::ThreadSocketHandler, this))); - if (!gArgs.GetBoolArg("-dnsseed", true)) + if (!gArgs.GetBoolArg("-dnsseed", DEFAULT_DNSSEED)) LogPrintf("DNS seeding disabled\n"); else threadDNSAddressSeed = std::thread(&TraceThread<std::function<void()> >, "dnsseed", std::function<void()>(std::bind(&CConnman::ThreadDNSAddressSeed, this))); @@ -2424,6 +2512,12 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) // Process messages threadMessageHandler = std::thread(&TraceThread<std::function<void()> >, "msghand", std::function<void()>(std::bind(&CConnman::ThreadMessageHandler, this))); + if (connOptions.m_i2p_accept_incoming && m_i2p_sam_session.get() != nullptr) { + threadI2PAcceptIncoming = + std::thread(&TraceThread<std::function<void()>>, "i2paccept", + std::function<void()>(std::bind(&CConnman::ThreadI2PAcceptIncoming, this))); + } + // Dump network addresses scheduler.scheduleEvery([this] { DumpAddresses(); }, DUMP_PEERS_INTERVAL); @@ -2471,6 +2565,9 @@ void CConnman::Interrupt() void CConnman::StopThreads() { + if (threadI2PAcceptIncoming.joinable()) { + threadI2PAcceptIncoming.join(); + } if (threadMessageHandler.joinable()) threadMessageHandler.join(); if (threadOpenConnections.joinable()) @@ -2567,9 +2664,7 @@ std::vector<CAddress> CConnman::GetAddresses(size_t max_addresses, size_t max_pc std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addresses, size_t max_pct) { - SOCKET socket; - WITH_LOCK(requestor.cs_hSocket, socket = requestor.hSocket); - auto local_socket_bytes = GetBindAddress(socket).GetAddrBytes(); + auto local_socket_bytes = requestor.addrBind.GetAddrBytes(); uint64_t cache_id = GetDeterministicRandomizer(RANDOMIZER_ID_ADDRCACHE) .Write(requestor.addr.GetNetwork()) .Write(local_socket_bytes.data(), local_socket_bytes.size()) @@ -2662,6 +2757,7 @@ bool CConnman::DisconnectNode(const std::string& strNode) { LOCK(cs_vNodes); if (CNode* pnode = FindNode(strNode)) { + LogPrint(BCLog::NET, "disconnect by address%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2674,6 +2770,7 @@ bool CConnman::DisconnectNode(const CSubNet& subnet) LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (subnet.Match(pnode->addr)) { + LogPrint(BCLog::NET, "disconnect by subnet%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->GetId()); pnode->fDisconnect = true; disconnected = true; } @@ -2691,6 +2788,7 @@ bool CConnman::DisconnectNode(NodeId id) LOCK(cs_vNodes); for(CNode* pnode : vNodes) { if (id == pnode->GetId()) { + LogPrint(BCLog::NET, "disconnect by id peer=%d; disconnecting\n", pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2798,12 +2896,12 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), addrBind(addrBindIn), + m_inbound_onion(inbound_onion), nKeyedNetGroup(nKeyedNetGroupIn), id(idIn), nLocalHostNonce(nLocalHostNonceIn), m_conn_type(conn_type_in), - nLocalServices(nLocalServicesIn), - m_inbound_onion(inbound_onion) + nLocalServices(nLocalServicesIn) { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); hSocket = hSocketIn; @@ -2844,6 +2942,9 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + } // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; @@ -2859,18 +2960,14 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; - if (pnode->nSendSize > nSendBufferMaxSize) - pnode->fPauseSend = true; + if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) - pnode->vSendMsg.push_back(std::move(msg.data)); + if (nMessageSize) pnode->vSendMsg.push_back(std::move(msg.data)); // If write queue empty, attempt "optimistic write" - if (optimisticSend == true) - nBytesSent = SocketSendData(*pnode); + if (optimisticSend) nBytesSent = SocketSendData(*pnode); } - if (nBytesSent) - RecordBytesSent(nBytesSent); + if (nBytesSent) RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -2913,3 +3010,31 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } + +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming) +{ + // Note: This function captures the message at the time of processing, + // not at socket receive/send time. + // This ensures that the messages are always in order from an application + // layer (processing) perspective. + auto now = GetTime<std::chrono::microseconds>(); + + // Windows folder names can not include a colon + std::string clean_addr = addr.ToString(); + std::replace(clean_addr.begin(), clean_addr.end(), ':', '_'); + + fs::path base_path = GetDataDir() / "message_capture" / clean_addr; + fs::create_directories(base_path); + + fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); + CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + + ser_writedata64(f, now.count()); + f.write(msg_type.data(), msg_type.length()); + for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { + f << '\0'; + } + uint32_t size = data.size(); + ser_writedata32(f, size); + f.write((const char*)data.data(), data.size()); +} @@ -14,12 +14,14 @@ #include <compat.h> #include <crypto/siphash.h> #include <hash.h> +#include <i2p.h> #include <net_permissions.h> #include <netaddress.h> #include <optional.h> #include <policy/feerate.h> #include <protocol.h> #include <random.h> +#include <span.h> #include <streams.h> #include <sync.h> #include <threadinterrupt.h> @@ -75,8 +77,12 @@ static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; +/** Number of file descriptors required for message capture **/ +static const int NUM_FDS_MESSAGE_CAPTURE = 1; static const bool DEFAULT_FORCEDNSSEED = false; +static const bool DEFAULT_DNSSEED = true; +static const bool DEFAULT_FIXEDSEEDS = true; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; @@ -111,7 +117,8 @@ struct CSerializedNetMsg * connection. Aside from INBOUND, all types are initiated by us. * * If adding or removing types, please update CONNECTION_TYPE_DOC in - * src/rpc/net.cpp. */ + * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in + * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ enum class ConnectionType { /** * Inbound connections are those initiated by a peer. This is the only @@ -122,7 +129,7 @@ enum class ConnectionType { /** * These are the default connections that we use to connect with the - * network. There is no restriction on what is relayed- by default we relay + * network. There is no restriction on what is relayed; by default we relay * blocks, addresses & transactions. We automatically attempt to open * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. */ @@ -130,8 +137,8 @@ enum class ConnectionType { /** - * We open manual connections to addresses that users explicitly inputted - * via the addnode RPC, or the -connect command line argument. Even if a + * We open manual connections to addresses that users explicitly requested + * via the addnode RPC or the -addnode/-connect configuration options. Even if a * manual connection is misbehaving, we do not automatically disconnect or * add it to our discouragement filter. */ @@ -150,7 +157,7 @@ enum class ConnectionType { * although in our codebase feeler connections encompass test-before-evict as well. * We make these connections approximately every FEELER_INTERVAL: * first we resolve previously found collisions if they exist (test-before-evict), - * otherwise connect to a node from the new table. + * otherwise we connect to a node from the new table. */ FEELER, @@ -174,6 +181,8 @@ enum class ConnectionType { ADDR_FETCH, }; +/** Convert ConnectionType enum to a string value */ +std::string ConnectionTypeAsString(ConnectionType conn_type); void Discover(); uint16_t GetListenPort(); @@ -189,7 +198,8 @@ enum }; bool IsPeerAddrLocalGood(CNode *pnode); -void AdvertiseLocal(CNode *pnode); +/** Returns a local address that we should advertise to this peer */ +Optional<CAddress> GetLocalAddrForPeer(CNode *pnode); /** * Mark a network as reachable or unreachable (no automatic connects to it) @@ -252,7 +262,6 @@ public: mapMsgCmdSize mapRecvBytesPerMsgCmd; NetPermissionFlags m_permissionFlags; int64_t m_ping_usec; - int64_t m_ping_wait_usec; int64_t m_min_ping_usec; CAmount minFeeFilter; // Our address, as reported by the peer @@ -264,11 +273,10 @@ public: // Network the peer connected through Network m_network; uint32_t m_mapped_as; - std::string m_conn_type_string; + ConnectionType m_conn_type; }; - /** Transport protocol agnostic message container. * Ideally it should only contain receive time, payload, * command and size. @@ -393,7 +401,7 @@ public: std::unique_ptr<TransportDeserializer> m_deserializer; std::unique_ptr<TransportSerializer> m_serializer; - // socket + NetPermissionFlags m_permissionFlags{PF_NONE}; std::atomic<ServiceFlags> nServices{NODE_NONE}; SOCKET hSocket GUARDED_BY(cs_hSocket); /** Total size of all vSendMsg entries */ @@ -422,6 +430,8 @@ public: const CAddress addr; // Bind address of our side of the connection const CAddress addrBind; + //! Whether this peer is an inbound onion, i.e. connected via our Tor onion service. + const bool m_inbound_onion; std::atomic<int> nVersion{0}; RecursiveMutex cs_SubVer; /** @@ -440,6 +450,7 @@ public: * messages, implying a preference to receive ADDRv2 instead of ADDR ones. */ std::atomic_bool m_wants_addrv2{false}; + /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the // next time DisconnectNodes() runs @@ -527,11 +538,6 @@ public: */ Network ConnectedThroughNetwork() const; -protected: - mapMsgCmdSize mapSendBytesPerMsgCmd GUARDED_BY(cs_vSend); - mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); - -public: // We selected peer as (compact blocks) high-bandwidth peer (BIP152) std::atomic<bool> m_bip152_highbandwidth_to{false}; // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) @@ -564,9 +570,8 @@ public: std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; std::chrono::microseconds nNextInvSend{0}; - RecursiveMutex cs_feeFilter; - // Minimum fee rate with which to filter inv's to this node - CAmount minFeeFilter GUARDED_BY(cs_feeFilter){0}; + /** Minimum fee rate with which to filter inv's to this node */ + std::atomic<CAmount> minFeeFilter{0}; CAmount lastSentFeeFilter{0}; int64_t nextSendTimeFeeFilter{0}; }; @@ -587,61 +592,18 @@ public: * in CConnman::AttemptToEvictConnection. */ std::atomic<int64_t> nLastTXTime{0}; - // Ping time measurement: - // The pong reply we're expecting, or 0 if no pong expected. - std::atomic<uint64_t> nPingNonceSent{0}; - /** When the last ping was sent, or 0 if no ping was ever sent */ - std::atomic<std::chrono::microseconds> m_ping_start{0us}; - // Last measured round-trip time. - std::atomic<int64_t> nPingUsecTime{0}; - // Best measured round-trip time. - std::atomic<int64_t> nMinPingUsecTime{std::numeric_limits<int64_t>::max()}; - // Whether a ping is requested. - std::atomic<bool> fPingQueued{false}; - - CNode(NodeId id, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion = false); + /** Last measured round-trip time. Used only for RPC/GUI stats/debugging.*/ + std::atomic<int64_t> m_last_ping_time{0}; + + /** Lowest measured round-trip time. Used as an inbound peer eviction + * criterium in CConnman::AttemptToEvictConnection. */ + std::atomic<int64_t> m_min_ping_time{std::numeric_limits<int64_t>::max()}; + + CNode(NodeId id, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion); ~CNode(); CNode(const CNode&) = delete; CNode& operator=(const CNode&) = delete; -private: - const NodeId id; - const uint64_t nLocalHostNonce; - const ConnectionType m_conn_type; - std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; - - //! Services offered to this peer. - //! - //! This is supplied by the parent CConnman during peer connection - //! (CConnman::ConnectNode()) from its attribute of the same name. - //! - //! This is const because there is no protocol defined for renegotiating - //! services initially offered to a peer. The set of local services we - //! offer should not change after initialization. - //! - //! An interesting example of this is NODE_NETWORK and initial block - //! download: a node which starts up from scratch doesn't have any blocks - //! to serve, but still advertises NODE_NETWORK because it will eventually - //! fulfill this role after IBD completes. P2P code is written in such a - //! way that it can gracefully handle peers who don't make good on their - //! service advertisements. - const ServiceFlags nLocalServices; - - NetPermissionFlags m_permissionFlags{ PF_NONE }; - std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread - - mutable RecursiveMutex cs_addrName; - std::string addrName GUARDED_BY(cs_addrName); - - // Our address, as reported by the peer - CService addrLocal GUARDED_BY(cs_addrLocal); - mutable RecursiveMutex cs_addrLocal; - - //! Whether this peer is an inbound onion, e.g. connected via our Tor onion service. - const bool m_inbound_onion{false}; - -public: - NodeId GetId() const { return id; } @@ -692,8 +654,6 @@ public: nRefCount--; } - - void AddAddressKnown(const CAddress& _addr) { assert(m_addr_known); @@ -725,7 +685,6 @@ public: } } - void AddKnownTx(const uint256& hash) { if (m_tx_relay != nullptr) { @@ -756,10 +715,48 @@ public: //! Sets the addrName only if it was not previously set void MaybeSetAddrName(const std::string& addrNameIn); - std::string ConnectionTypeAsString() const; + std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } - /** Whether this peer is an inbound onion, e.g. connected via our Tor onion service. */ - bool IsInboundOnion() const { return m_inbound_onion; } + /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ + void PongReceived(std::chrono::microseconds ping_time) { + m_last_ping_time = count_microseconds(ping_time); + m_min_ping_time = std::min(m_min_ping_time.load(), count_microseconds(ping_time)); + } + +private: + const NodeId id; + const uint64_t nLocalHostNonce; + const ConnectionType m_conn_type; + std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; + + //! Services offered to this peer. + //! + //! This is supplied by the parent CConnman during peer connection + //! (CConnman::ConnectNode()) from its attribute of the same name. + //! + //! This is const because there is no protocol defined for renegotiating + //! services initially offered to a peer. The set of local services we + //! offer should not change after initialization. + //! + //! An interesting example of this is NODE_NETWORK and initial block + //! download: a node which starts up from scratch doesn't have any blocks + //! to serve, but still advertises NODE_NETWORK because it will eventually + //! fulfill this role after IBD completes. P2P code is written in such a + //! way that it can gracefully handle peers who don't make good on their + //! service advertisements. + const ServiceFlags nLocalServices; + + std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread + + mutable RecursiveMutex cs_addrName; + std::string addrName GUARDED_BY(cs_addrName); + + // Our address, as reported by the peer + CService addrLocal GUARDED_BY(cs_addrLocal); + mutable RecursiveMutex cs_addrLocal; + + mapMsgCmdSize mapSendBytesPerMsgCmd GUARDED_BY(cs_vSend); + mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); }; /** @@ -768,11 +765,30 @@ public: class NetEventsInterface { public: - virtual bool ProcessMessages(CNode* pnode, std::atomic<bool>& interrupt) = 0; - virtual bool SendMessages(CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_sendProcessing) = 0; + /** Initialize a peer (setup state, queue any initial messages) */ virtual void InitializeNode(CNode* pnode) = 0; + + /** Handle removal of a peer (clear state) */ virtual void FinalizeNode(const CNode& node, bool& update_connection_time) = 0; + /** + * Process protocol messages received from a given node + * + * @param[in] pnode The node which we have received messages from. + * @param[in] interrupt Interrupt condition for processing threads + * @return True if there is more work to be done + */ + virtual bool ProcessMessages(CNode* pnode, std::atomic<bool>& interrupt) = 0; + + /** + * Send queued protocol messages to a given node. + * + * @param[in] pnode The node which we are sending messages to. + * @return True if there is more work to be done + */ + virtual bool SendMessages(CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_sendProcessing) = 0; + + protected: /** * Protected destructor so that instances can only be deleted by derived classes. @@ -816,6 +832,7 @@ public: std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; std::vector<bool> m_asmap; + bool m_i2p_accept_incoming; }; void Init(const Options& connOptions) { @@ -946,6 +963,19 @@ public: bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); + /** + * Attempts to open a connection. Currently only used from tests. + * + * @param[in] address Address of node to try connecting to + * @param[in] conn_type ConnectionType::OUTBOUND or ConnectionType::BLOCK_RELAY + * @return bool Returns false if there are no available + * slots for this connection: + * - conn_type not a supported ConnectionType + * - Max total outbound connection capacity filled + * - Max connection capacity for type is filled + */ + bool AddConnection(const std::string& address, ConnectionType conn_type); + size_t GetNodeCount(NumConnections num); void GetNodeStats(std::vector<CNodeStats>& vstats); bool DisconnectNode(const std::string& node); @@ -995,6 +1025,9 @@ public: void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } + /** Return true if the peer has been connected for long enough to do inactivity checks. */ + bool RunInactivityChecks(const CNode& node) const; + private: struct ListenSocket { public: @@ -1017,10 +1050,26 @@ private: void ProcessAddrFetch(); void ThreadOpenConnections(std::vector<std::string> connect); void ThreadMessageHandler(); + void ThreadI2PAcceptIncoming(); void AcceptConnection(const ListenSocket& hListenSocket); + + /** + * Create a `CNode` object from a socket that has just been accepted and add the node to + * the `vNodes` member. + * @param[in] hSocket Connected socket to communicate with the peer. + * @param[in] permissionFlags The peer's permissions. + * @param[in] addr_bind The address and port at our side of the connection. + * @param[in] addr The address and port at the peer's side of the connection. + */ + void CreateNodeFromAcceptedSocket(SOCKET hSocket, + NetPermissionFlags permissionFlags, + const CAddress& addr_bind, + const CAddress& addr); + void DisconnectNodes(); void NotifyNumConnectionsChanged(); - void InactivityCheck(CNode *pnode) const; + /** Return true if the peer is inactive and should be disconnected. */ + bool InactivityCheck(const CNode& node) const; bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); void SocketHandler(); @@ -1175,13 +1224,26 @@ private: Mutex mutexMsgProc; std::atomic<bool> flagInterruptMsgProc{false}; + /** + * This is signaled when network activity should cease. + * A pointer to it is saved in `m_i2p_sam_session`, so make sure that + * the lifetime of `interruptNet` is not shorter than + * the lifetime of `m_i2p_sam_session`. + */ CThreadInterrupt interruptNet; + /** + * I2P SAM session. + * Used to accept incoming and make outgoing I2P connections. + */ + std::unique_ptr<i2p::sam::Session> m_i2p_sam_session; + std::thread threadDNSAddressSeed; std::thread threadSocketHandler; std::thread threadOpenAddedConnections; std::thread threadOpenConnections; std::thread threadMessageHandler; + std::thread threadI2PAcceptIncoming; /** flag for deciding to connect to an extra outbound peer, * in excess of m_max_outbound_full_relay @@ -1215,11 +1277,14 @@ inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; } +/** Dump binary message to file, with timestamp */ +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming); + struct NodeEvictionCandidate { NodeId id; int64_t nTimeConnected; - int64_t nMinPingUsecTime; + int64_t m_min_ping_time; int64_t nLastBlockTime; int64_t nLastTXTime; bool fRelevantServices; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index dc9b051ddd..c97f7ced46 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -26,6 +26,7 @@ #include <streams.h> #include <tinyformat.h> #include <txmempool.h> +#include <txrequest.h> #include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> #include <util/system.h> @@ -168,6 +169,204 @@ void EraseOrphansFor(NodeId peer); // Internal stuff namespace { +/** Blocks that are in flight, and that are in the queue to be downloaded. */ +struct QueuedBlock { + uint256 hash; + const CBlockIndex* pindex; //!< Optional. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads +}; + +/** + * Data structure for an individual peer. This struct is not protected by + * cs_main since it does not contain validation-critical data. + * + * Memory is owned by shared pointers and this object is destructed when + * the refcount drops to zero. + * + * Mutexes inside this struct must not be held when locking m_peer_mutex. + * + * TODO: move most members from CNodeState to this structure. + * TODO: move remaining application-layer data members from CNode to this structure. + */ +struct Peer { + /** Same id as the CNode object for this peer */ + const NodeId m_id{0}; + + /** Protects misbehavior data members */ + Mutex m_misbehavior_mutex; + /** Accumulated misbehavior score for this peer */ + int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; + /** Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). */ + bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; + + /** Protects block inventory data members */ + Mutex m_block_inv_mutex; + /** List of blocks that we'll announce via an `inv` message. + * There is no final sorting before sending, as they are always sent + * immediately and in the order requested. */ + std::vector<uint256> m_blocks_for_inv_relay GUARDED_BY(m_block_inv_mutex); + /** Unfiltered list of blocks that we'd like to announce via a `headers` + * message. If we can't announce via a `headers` message, we'll fall back to + * announcing via `inv`. */ + std::vector<uint256> m_blocks_for_headers_relay GUARDED_BY(m_block_inv_mutex); + /** The final block hash that we sent in an `inv` message to this peer. + * When the peer requests this block, we send an `inv` message to trigger + * the peer to request the next sequence of block hashes. + * Most peers use headers-first syncing, which doesn't use this mechanism */ + uint256 m_continuation_block GUARDED_BY(m_block_inv_mutex) {}; + + /** This peer's reported block height when we connected */ + std::atomic<int> m_starting_height{-1}; + + /** The pong reply we're expecting, or 0 if no pong expected. */ + std::atomic<uint64_t> m_ping_nonce_sent{0}; + /** When the last ping was sent, or 0 if no ping was ever sent */ + std::atomic<std::chrono::microseconds> m_ping_start{0us}; + /** Whether a ping has been requested by the user */ + std::atomic<bool> m_ping_queued{false}; + + /** Set of txids to reconsider once their parent transactions have been accepted **/ + std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); + + /** Protects m_getdata_requests **/ + Mutex m_getdata_requests_mutex; + /** Work queue of items requested by this peer **/ + std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); + + explicit Peer(NodeId id) : m_id(id) {} +}; + +using PeerRef = std::shared_ptr<Peer>; + +class PeerManagerImpl final : public PeerManager +{ +public: + PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs); + + /** Overridden from CValidationInterface. */ + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; + void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; + void BlockChecked(const CBlock& block, const BlockValidationState& state) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; + + /** Implement NetEventsInterface */ + void InitializeNode(CNode* pnode) override; + void FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) override; + bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override; + bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); + + /** Implement PeerManager */ + void CheckForStaleTipAndEvictPeers() override; + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) override; + bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } + void SendPings() override; + void SetBestHeight(int height) override { m_best_height = height; }; + void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) override; + void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override; + +private: + /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ + void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ + void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ + void ReattemptInitialBroadcast(CScheduler& scheduler) const; + + /** Get a shared pointer to the Peer object. + * May return an empty shared_ptr if the Peer object can't be found. */ + PeerRef GetPeerRef(NodeId id) const; + + /** Get a shared pointer to the Peer object and remove it from m_peer_map. + * May return an empty shared_ptr if the Peer object can't be found. */ + PeerRef RemovePeer(NodeId id); + + /** + * Potentially mark a node discouraged based on the contents of a BlockValidationState object + * + * @param[in] via_compact_block this bool is passed in because net_processing should + * punish peers differently depending on whether the data was provided in a compact + * block message or not. If the compact block had a valid header, but contained invalid + * txs, the peer should not be punished. See BIP 152. + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message = ""); + + /** + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = ""); + + /** Maybe disconnect a peer and discourage future connections from its address. + * + * @param[in] pnode The node to check. + * @param[in] peer The peer object to check. + * @return True if the peer was marked for disconnection in this function + */ + bool MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer); + + void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); + /** Process a single headers message from a peer. */ + void ProcessHeadersMessage(CNode& pfrom, const Peer& peer, + const std::vector<CBlockHeader>& headers, + bool via_compact_block); + + void SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req); + + /** Register with TxRequestTracker that an INV has been received from a + * peer. The announcement parameters are decided in PeerManager and then + * passed to TxRequestTracker. */ + void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + /** Send a version message to a peer */ + void PushNodeVersion(CNode& pnode, int64_t nTime); + + /** Send a ping message every PING_INTERVAL or if requested via RPC. May + * mark the peer to be disconnected if a ping has timed out. */ + void MaybeSendPing(CNode& node_to, Peer& peer); + + const CChainParams& m_chainparams; + CConnman& m_connman; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ + BanMan* const m_banman; + ChainstateManager& m_chainman; + CTxMemPool& m_mempool; + TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + + /** The height of the best chain */ + std::atomic<int> m_best_height{-1}; + + int64_t m_stale_tip_check_time; //!< Next time to check for stale tip + + /** Whether this node is running in blocks only mode */ + const bool m_ignore_incoming_txs; + + /** Whether we've completed initial sync yet, for determining when to turn + * on extra block-relay-only peers. */ + bool m_initial_sync_finished{false}; + + /** Protects m_peer_map. This mutex must not be locked while holding a lock + * on any of the mutexes inside a Peer object. */ + mutable Mutex m_peer_mutex; + /** + * Map of all Peer objects, keyed by peer id. This map is protected + * by the m_peer_mutex. Once a shared pointer reference is + * taken, the lock may be released. Individual fields are protected by + * their own locks. + */ + std::map<NodeId, PeerRef> m_peer_map GUARDED_BY(m_peer_mutex); + /** Number of nodes with fSyncStarted. */ int nSyncStarted GUARDED_BY(cs_main) = 0; @@ -179,6 +378,14 @@ namespace { */ std::map<uint256, std::pair<NodeId, bool>> mapBlockSource GUARDED_BY(cs_main); + /** Number of peers with wtxid relay. */ + int m_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + + /** Number of outbound peers with m_chain_sync.m_protect. */ + int m_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; + + bool AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** * Filter for transactions that were recently rejected by * AcceptToMemoryPool. These are not rerequested until the chain tip @@ -221,35 +428,36 @@ namespace { * We use this to avoid requesting transactions that have already been * confirnmed. */ - Mutex g_cs_recent_confirmed_transactions; - std::unique_ptr<CRollingBloomFilter> g_recent_confirmed_transactions GUARDED_BY(g_cs_recent_confirmed_transactions); - - /** Blocks that are in flight, and that are in the queue to be downloaded. */ - struct QueuedBlock { - uint256 hash; - const CBlockIndex* pindex; //!< Optional. - bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. - std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads - }; - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); + Mutex m_recent_confirmed_transactions_mutex; + std::unique_ptr<CRollingBloomFilter> m_recent_confirmed_transactions GUARDED_BY(m_recent_confirmed_transactions_mutex); - /** Stack of nodes which we have set to announce using compact blocks */ - std::list<NodeId> lNodesAnnouncingHeaderAndIDs GUARDED_BY(cs_main); + /* Returns a bool indicating whether we requested this block. + * Also used if a block was /not/ received and timed out or started with another peer + */ + bool MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Number of preferable block download peers. */ - int nPreferredDownload GUARDED_BY(cs_main) = 0; + /* Mark a block as in flight + * Returns false, still setting pit, if the block was already in flight from the same peer + * pit will only be valid as long as the same cs_main lock is being held + */ + bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = nullptr, std::list<QueuedBlock>::iterator** pit = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Number of peers from which we're downloading blocks. */ - int nPeersWithValidatedDownloads GUARDED_BY(cs_main) = 0; + bool TipMayBeStale() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Number of peers with wtxid relay. */ - int g_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + /** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has + * at most count entries. + */ + void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Number of outbound peers with m_chain_sync.m_protect. */ - int g_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; + std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); /** When our tip was last updated. */ - std::atomic<int64_t> g_last_tip_update(0); + std::atomic<int64_t> m_last_tip_update{0}; + + /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ + CTransactionRef FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main); + + void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex); /** Relay map (txid or wtxid -> CTransactionRef) */ typedef std::map<uint256, CTransactionRef> MapRelay; @@ -257,6 +465,28 @@ namespace { /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration GUARDED_BY(cs_main); + /** + * When a peer sends us a valid block, instruct it to announce blocks to us + * using CMPCTBLOCK if possible by adding its nodeid to the end of + * lNodesAnnouncingHeaderAndIDs, and keeping that list under a certain size by + * removing the first element if necessary. + */ + void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Stack of nodes which we have set to announce using compact blocks */ + std::list<NodeId> lNodesAnnouncingHeaderAndIDs GUARDED_BY(cs_main); + + /** Number of peers from which we're downloading blocks. */ + int nPeersWithValidatedDownloads GUARDED_BY(cs_main) = 0; + +}; +} // namespace + +namespace { + + /** Number of preferable block download peers. */ + int nPreferredDownload GUARDED_BY(cs_main) = 0; + struct IteratorComparator { template<typename I> @@ -429,9 +659,8 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } -// Returns a bool indicating whether we requested this block. -// Also used if a block was /not/ received and timed out or started with another peer -static bool MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +bool PeerManagerImpl::MarkBlockAsReceived(const uint256& hash) +{ std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { CNodeState *state = State(itInFlight->second.first); @@ -454,9 +683,8 @@ static bool MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs return false; } -// returns false, still setting pit, if the block was already in flight from the same peer -// pit will only be valid as long as the same cs_main lock is being held -static bool MarkBlockAsInFlight(CTxMemPool& mempool, NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = nullptr, std::list<QueuedBlock>::iterator** pit = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +bool PeerManagerImpl::MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex, std::list<QueuedBlock>::iterator** pit) +{ CNodeState *state = State(nodeid); assert(state != nullptr); @@ -473,7 +701,7 @@ static bool MarkBlockAsInFlight(CTxMemPool& mempool, NodeId nodeid, const uint25 MarkBlockAsReceived(hash); std::list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), - {hash, pindex, pindex != nullptr, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&mempool) : nullptr)}); + {hash, pindex, pindex != nullptr, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&m_mempool) : nullptr)}); state->nBlocksInFlight++; state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; if (state->nBlocksInFlight == 1) { @@ -495,7 +723,7 @@ static void ProcessBlockAvailability(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_ assert(state != nullptr); if (!state->hashLastUnknownBlock.IsNull()) { - const CBlockIndex* pindex = LookupBlockIndex(state->hashLastUnknownBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(state->hashLastUnknownBlock); if (pindex && pindex->nChainWork > 0) { if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { state->pindexBestKnownBlock = pindex; @@ -512,7 +740,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV ProcessBlockAvailability(nodeid); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex && pindex->nChainWork > 0) { // An actually better block was announced. if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { @@ -524,13 +752,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV } } -/** - * When a peer sends us a valid block, instruct it to announce blocks to us - * using CMPCTBLOCK if possible by adding its nodeid to the end of - * lNodesAnnouncingHeaderAndIDs, and keeping that list under a certain size by - * removing the first element if necessary. - */ -static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); @@ -546,21 +768,21 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connma return; } } - connman.ForNode(nodeid, [&connman](CNode* pfrom) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + m_connman.ForNode(nodeid, [this](CNode* pfrom) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); + m_connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [this, nCMPCTBLOCKVersion](CNode* pnodeStop){ + m_connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); // save BIP152 bandwidth state: we select peer to be low-bandwidth pnodeStop->m_bip152_highbandwidth_to = false; return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); + m_connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); // save BIP152 bandwidth state: we select peer to be high-bandwidth pfrom->m_bip152_highbandwidth_to = true; lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); @@ -569,13 +791,14 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connma } } -static bool TipMayBeStale(const Consensus::Params &consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::TipMayBeStale() { AssertLockHeld(cs_main); - if (g_last_tip_update == 0) { - g_last_tip_update = GetTime(); + const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); + if (m_last_tip_update == 0) { + m_last_tip_update = GetTime(); } - return g_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); + return m_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); } static bool CanDirectFetch(const Consensus::Params &consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -592,9 +815,7 @@ static bool PeerHasHeader(CNodeState *state, const CBlockIndex *pindex) EXCLUSIV return false; } -/** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has - * at most count entries. */ -static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) { if (count == 0) return; @@ -623,6 +844,7 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) return; + const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); std::vector<const CBlockIndex*> vToFetch; const CBlockIndex *pindexWalk = state->pindexLastCommonBlock; // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last @@ -683,7 +905,7 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec } // namespace -void PeerManager::PushNodeVersion(CNode& pnode, int64_t nTime) +void PeerManagerImpl::PushNodeVersion(CNode& pnode, int64_t nTime) { // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this @@ -699,17 +921,18 @@ void PeerManager::PushNodeVersion(CNode& pnode, int64_t nTime) CAddress(CService(), addr.nServices); CAddress addrMe = CAddress(CService(), nLocalNodeServices); + const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr; m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr)); + nonce, strSubVersion, nNodeStartingHeight, tx_relay)); if (fLogIPs) { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), tx_relay, nodeid); } else { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), tx_relay, nodeid); } } -void PeerManager::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) +void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) { AssertLockHeld(::cs_main); // For m_txrequest NodeId nodeid = node.GetId(); @@ -729,7 +952,7 @@ void PeerManager::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std auto delay = std::chrono::microseconds{0}; const bool preferred = state->fPreferredDownload; if (!preferred) delay += NONPREF_PEER_TX_DELAY; - if (!gtxid.IsWtxid() && g_wtxid_relay_peers > 0) delay += TXID_RELAY_DELAY; + if (!gtxid.IsWtxid() && m_wtxid_relay_peers > 0) delay += TXID_RELAY_DELAY; const bool overloaded = !node.HasPermission(PF_RELAY) && m_txrequest.CountInFlight(nodeid) >= MAX_PEER_TX_REQUEST_IN_FLIGHT; if (overloaded) delay += OVERLOADED_PEER_TX_DELAY; @@ -745,7 +968,8 @@ void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) if (state) state->m_last_block_announcement = time_in_seconds; } -void PeerManager::InitializeNode(CNode *pnode) { +void PeerManagerImpl::InitializeNode(CNode *pnode) +{ CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); NodeId nodeid = pnode->GetId(); @@ -764,7 +988,7 @@ void PeerManager::InitializeNode(CNode *pnode) { } } -void PeerManager::ReattemptInitialBroadcast(CScheduler& scheduler) const +void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) const { std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); @@ -785,7 +1009,8 @@ void PeerManager::ReattemptInitialBroadcast(CScheduler& scheduler) const scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } -void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { +void PeerManagerImpl::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) +{ NodeId nodeid = node.GetId(); fUpdateConnectionTime = false; LOCK(cs_main); @@ -820,10 +1045,10 @@ void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { nPreferredDownload -= state->fPreferredDownload; nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); assert(nPeersWithValidatedDownloads >= 0); - g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; - assert(g_outbound_peers_with_protect_from_disconnect >= 0); - g_wtxid_relay_peers -= state->m_wtxid_relay; - assert(g_wtxid_relay_peers >= 0); + m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; + assert(m_outbound_peers_with_protect_from_disconnect >= 0); + m_wtxid_relay_peers -= state->m_wtxid_relay; + assert(m_wtxid_relay_peers >= 0); mapNodeState.erase(nodeid); @@ -832,21 +1057,21 @@ void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { assert(mapBlocksInFlight.empty()); assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); - assert(g_outbound_peers_with_protect_from_disconnect == 0); - assert(g_wtxid_relay_peers == 0); + assert(m_outbound_peers_with_protect_from_disconnect == 0); + assert(m_wtxid_relay_peers == 0); assert(m_txrequest.Size() == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } -PeerRef PeerManager::GetPeerRef(NodeId id) const +PeerRef PeerManagerImpl::GetPeerRef(NodeId id) const { LOCK(m_peer_mutex); auto it = m_peer_map.find(id); return it != m_peer_map.end() ? it->second : nullptr; } -PeerRef PeerManager::RemovePeer(NodeId id) +PeerRef PeerManagerImpl::RemovePeer(NodeId id) { PeerRef ret; LOCK(m_peer_mutex); @@ -858,7 +1083,8 @@ PeerRef PeerManager::RemovePeer(NodeId id) return ret; } -bool PeerManager::GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { +bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) +{ { LOCK(cs_main); CNodeState* state = State(nodeid); @@ -875,6 +1101,18 @@ bool PeerManager::GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; stats.m_starting_height = peer->m_starting_height; + // It is common for nodes with good ping times to suddenly become lagged, + // due to a new block arriving or other large transfer. + // Merely reporting pingtime might fool the caller into thinking the node was still responsive, + // since pingtime does not update until the ping is complete, which might take a while. + // So, if a ping is taking an unusually long time in flight, + // the caller can immediately detect that this is happening. + std::chrono::microseconds ping_wait{0}; + if ((0 != peer->m_ping_nonce_sent) && (0 != peer->m_ping_start.load().count())) { + ping_wait = GetTime<std::chrono::microseconds>() - peer->m_ping_start.load(); + } + + stats.m_ping_wait_usec = count_microseconds(ping_wait); return true; } @@ -1015,7 +1253,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) return nEvicted; } -void PeerManager::Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) +void PeerManagerImpl::Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) { assert(howmuch > 0); @@ -1033,8 +1271,8 @@ void PeerManager::Misbehaving(const NodeId pnode, const int howmuch, const std:: } } -bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, - bool via_compact_block, const std::string& message) +bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message) { switch (state.GetResult()) { case BlockValidationResult::BLOCK_RESULT_UNSET: @@ -1083,7 +1321,7 @@ bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationSt return false; } -bool PeerManager::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) +bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) { switch (state.GetResult()) { case TxValidationResult::TX_RESULT_UNSET: @@ -1129,9 +1367,16 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, - CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, - bool ignore_incoming_txs) +std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs) +{ + return std::make_unique<PeerManagerImpl>(chainparams, connman, banman, scheduler, chainman, pool, ignore_incoming_txs); +} + +PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs) : m_chainparams(chainparams), m_connman(connman), m_banman(banman), @@ -1152,7 +1397,7 @@ PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, Ban // The false positive rate of 1/1M should come out to less than 1 // transaction per day that would be inadvertently ignored (which is the // same probability that we have in the reject filter). - g_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); + m_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); // Stale tip checking and peer eviction are on two different timers, but we // don't want them to get out of sync due to drift in the scheduler, so we @@ -1171,7 +1416,7 @@ PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, Ban * block, remember the recently confirmed transactions, and delete tracked * announcements for them. Also save the time of the last tip update. */ -void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { { LOCK(g_cs_orphans); @@ -1202,14 +1447,14 @@ void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, co LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } - g_last_tip_update = GetTime(); + m_last_tip_update = GetTime(); } { - LOCK(g_cs_recent_confirmed_transactions); + LOCK(m_recent_confirmed_transactions_mutex); for (const auto& ptx : pblock->vtx) { - g_recent_confirmed_transactions->insert(ptx->GetHash()); + m_recent_confirmed_transactions->insert(ptx->GetHash()); if (ptx->GetHash() != ptx->GetWitnessHash()) { - g_recent_confirmed_transactions->insert(ptx->GetWitnessHash()); + m_recent_confirmed_transactions->insert(ptx->GetWitnessHash()); } } } @@ -1222,7 +1467,7 @@ void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, co } } -void PeerManager::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) +void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) { // To avoid relay problems with transactions that were previously // confirmed, clear our filter of recently confirmed transactions whenever @@ -1232,8 +1477,8 @@ void PeerManager::BlockDisconnected(const std::shared_ptr<const CBlock> &block, // block's worth of transactions in it, but that should be fine, since // presumably the most common case of relaying a confirmed transaction // should be just after a new block containing it is found. - LOCK(g_cs_recent_confirmed_transactions); - g_recent_confirmed_transactions->reset(); + LOCK(m_recent_confirmed_transactions_mutex); + m_recent_confirmed_transactions->reset(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -1247,7 +1492,8 @@ static bool fWitnessesPresentInMostRecentCompactBlock GUARDED_BY(cs_most_recent_ * Maintain state about the best-seen block and fast-announce a compact block * to compatible peers. */ -void PeerManager::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { +void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) +{ std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); const CNetMsgMaker msgMaker(PROTOCOL_VERSION); @@ -1294,9 +1540,9 @@ void PeerManager::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ * Update our best height and announce any block hashes which weren't previously * in ::ChainActive() to our peers. */ -void PeerManager::UpdatedBlockTip(const CBlockIndex* pindexNew, const CBlockIndex* pindexFork, bool fInitialDownload) +void PeerManagerImpl::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - m_best_height = pindexNew->nHeight; + SetBestHeight(pindexNew->nHeight); SetServiceFlagsIBDCache(!fInitialDownload); // Don't relay inventory during initial block download. @@ -1333,7 +1579,8 @@ void PeerManager::UpdatedBlockTip(const CBlockIndex* pindexNew, const CBlockInde * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ -void PeerManager::BlockChecked(const CBlock& block, const BlockValidationState& state) { +void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationState& state) +{ LOCK(cs_main); const uint256 hash(block.GetHash()); @@ -1356,7 +1603,7 @@ void PeerManager::BlockChecked(const CBlock& block, const BlockValidationState& !::ChainstateActive().IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, m_connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first); } } if (it != mapBlockSource.end()) @@ -1369,7 +1616,7 @@ void PeerManager::BlockChecked(const CBlock& block, const BlockValidationState& // -bool static AlreadyHaveTx(const GenTxid& gtxid, const CTxMemPool& mempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) { assert(recentRejects); if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { @@ -1393,16 +1640,22 @@ bool static AlreadyHaveTx(const GenTxid& gtxid, const CTxMemPool& mempool) EXCLU } { - LOCK(g_cs_recent_confirmed_transactions); - if (g_recent_confirmed_transactions->contains(hash)) return true; + LOCK(m_recent_confirmed_transactions_mutex); + if (m_recent_confirmed_transactions->contains(hash)) return true; } - return recentRejects->contains(hash) || mempool.exists(gtxid); + return recentRejects->contains(hash) || m_mempool.exists(gtxid); } bool static AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - return LookupBlockIndex(block_hash) != nullptr; + return g_chainman.m_blockman.LookupBlockIndex(block_hash) != nullptr; +} + +void PeerManagerImpl::SendPings() +{ + LOCK(m_peer_mutex); + for(auto& it : m_peer_map) it.second->m_ping_queued = true; } void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) @@ -1492,7 +1745,7 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch bool need_activate_chain = false; { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { if (pindex->HaveTxsDownloaded() && !pindex->IsValid(BLOCK_VALID_SCRIPTS) && pindex->IsValid(BLOCK_VALID_TREE)) { @@ -1507,13 +1760,13 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch } // release cs_main before calling ActivateBestChain if (need_activate_chain) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { send = BlockRequestAllowed(pindex, consensusParams); if (!send) { @@ -1631,10 +1884,9 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch } } -//! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). -static CTransactionRef FindTxForGetData(const CTxMemPool& mempool, const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) +CTransactionRef PeerManagerImpl::FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) { - auto txinfo = mempool.info(gtxid); + auto txinfo = m_mempool.info(gtxid); if (txinfo.tx) { // If a TX could have been INVed in reply to a MEMPOOL request, // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request @@ -1659,7 +1911,7 @@ static CTransactionRef FindTxForGetData(const CTxMemPool& mempool, const CNode& return {}; } -void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainparams, CConnman& connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex) +void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) { AssertLockNotHeld(cs_main); @@ -1688,17 +1940,17 @@ void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainpa continue; } - CTransactionRef tx = FindTxForGetData(mempool, pfrom, ToGenTxid(inv), mempool_req, now); + CTransactionRef tx = FindTxForGetData(pfrom, ToGenTxid(inv), mempool_req, now); if (tx) { // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); - mempool.RemoveUnbroadcastTx(tx->GetHash()); + m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + m_mempool.RemoveUnbroadcastTx(tx->GetHash()); // As we're going to send tx, make sure its unconfirmed parents are made requestable. std::vector<uint256> parent_ids_to_add; { - LOCK(mempool.cs); - auto txiter = mempool.GetIter(tx->GetHash()); + LOCK(m_mempool.cs); + auto txiter = m_mempool.GetIter(tx->GetHash()); if (txiter) { const CTxMemPoolEntry::Parents& parents = (*txiter)->GetMemPoolParentsConst(); parent_ids_to_add.reserve(parents.size()); @@ -1726,7 +1978,7 @@ void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainpa if (it != peer.m_getdata_requests.end() && !pfrom.fPauseSend) { const CInv &inv = *it++; if (inv.IsGenBlkMsg()) { - ProcessGetBlockData(pfrom, peer, chainparams, inv, connman); + ProcessGetBlockData(pfrom, peer, m_chainparams, inv, m_connman); } // else: If the first item on the queue is an unknown type, we erase it // and continue processing the queue on the next call. @@ -1749,7 +2001,7 @@ void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainpa // In normal operation, we often send NOTFOUND messages for parents of // transactions that we relay; if a peer is missing a parent, they may // assume we have them and request the parents from us. - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); } } @@ -1761,7 +2013,8 @@ static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_ma return nFetchFlags; } -void PeerManager::SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req) { +void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req) +{ BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { @@ -1776,9 +2029,9 @@ void PeerManager::SendBlockTransactions(CNode& pfrom, const CBlock& block, const m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, - const std::vector<CBlockHeader>& headers, - bool via_compact_block) +void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, + const std::vector<CBlockHeader>& headers, + bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); size_t nCount = headers.size(); @@ -1802,7 +2055,7 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // don't connect before giving DoS points // - Once a headers message is received that is valid and does connect, // nUnconnectingHeaders gets reset back to 0. - if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + if (!g_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", @@ -1832,7 +2085,7 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // If we don't have the last header, then they'll have given us // something new (if these headers are valid). - if (!LookupBlockIndex(hashLastBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { received_new_header = true; } } @@ -1907,7 +2160,7 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, } uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(m_mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex); + MarkBlockAsInFlight(pfrom.GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", pindex->GetBlockHash().ToString(), pfrom.GetId()); } @@ -1951,10 +2204,10 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // thus always subject to eviction under the bad/lagging chain logic. // See ChainSyncTimeoutState. if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { - if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= ::ChainActive().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= ::ChainActive().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); nodestate->m_chain_sync.m_protect = true; - ++g_outbound_peers_with_protect_from_disconnect; + ++m_outbound_peers_with_protect_from_disconnect; } } } @@ -1970,7 +2223,7 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, * may be added to if accepting an orphan causes its children to be * reconsidered. */ -void PeerManager::ProcessOrphanTx(std::set<uint256>& orphan_work_set) +void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) { AssertLockHeld(cs_main); AssertLockHeld(g_cs_orphans); @@ -1983,10 +2236,10 @@ void PeerManager::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (orphan_it == mapOrphanTransactions.end()) continue; const CTransactionRef porphanTx = orphan_it->second.tx; - TxValidationState state; - std::list<CTransactionRef> removed_txn; + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), m_mempool, porphanTx, false /* bypass_limits */); + const TxValidationState& state = result.m_state; - if (AcceptToMemoryPool(m_mempool, state, porphanTx, &removed_txn, false /* bypass_limits */)) { + if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, porphanTx->GetWitnessHash(), m_connman); for (unsigned int i = 0; i < porphanTx->vout.size(); i++) { @@ -1998,7 +2251,7 @@ void PeerManager::ProcessOrphanTx(std::set<uint256>& orphan_work_set) } } EraseOrphanTx(orphanHash); - for (const CTransactionRef& removedTx : removed_txn) { + for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } break; @@ -2084,7 +2337,7 @@ static bool PrepareBlockFilterRequest(CNode& peer, const CChainParams& chain_par { LOCK(cs_main); - stop_index = LookupBlockIndex(stop_hash); + stop_index = g_chainman.m_blockman.LookupBlockIndex(stop_hash); // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { @@ -2267,9 +2520,9 @@ static void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv, const CChainPar connman.PushMessage(&peer, std::move(msg)); } -void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - const std::chrono::microseconds time_received, - const std::atomic<bool>& interruptMsgProc) +void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, + const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); @@ -2294,6 +2547,9 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat bool fRelay = true; vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; + if (nTime < 0) { + nTime = 0; + } nServices = ServiceFlags(nServiceInt); if (!pfrom.IsInboundConn()) { @@ -2448,9 +2704,9 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (fLogIPs) remoteAddr = ", peeraddr=" + pfrom.addr.ToString(); - LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d%s\n", cleanSubVer, pfrom.nVersion, - peer->m_starting_height, addrMe.ToString(), pfrom.GetId(), + peer->m_starting_height, addrMe.ToString(), fRelay, pfrom.GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -2465,6 +2721,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // Feeler connections exist only to verify if address is online. if (pfrom.IsFeelerConn()) { + LogPrint(BCLog::NET, "feeler connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; } return; @@ -2480,7 +2737,10 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); if (msg_type == NetMsgType::VERACK) { - if (pfrom.fSuccessfullyConnected) return; + if (pfrom.fSuccessfullyConnected) { + LogPrint(BCLog::NET, "ignoring redundant verack message from peer=%d\n", pfrom.GetId()); + return; + } if (!pfrom.IsInboundConn()) { LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s (%s)\n", @@ -2546,12 +2806,12 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } - // Feature negotiation of wtxidrelay must happen between VERSION and VERACK - // to avoid relay problems from switching after a connection is up. + // BIP339 defines feature negotiation of wtxidrelay, which must happen between + // VERSION and VERACK to avoid relay problems from switching after a connection is up. if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send wtxidrelay message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a wtxidrelay message after VERACK. + LogPrint(BCLog::NET, "wtxidrelay received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2559,16 +2819,22 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LOCK(cs_main); if (!State(pfrom.GetId())->m_wtxid_relay) { State(pfrom.GetId())->m_wtxid_relay = true; - g_wtxid_relay_peers++; + m_wtxid_relay_peers++; + } else { + LogPrint(BCLog::NET, "ignoring duplicate wtxidrelay from peer=%d\n", pfrom.GetId()); } + } else { + LogPrint(BCLog::NET, "ignoring wtxidrelay due to old common version=%d from peer=%d\n", pfrom.GetCommonVersion(), pfrom.GetId()); } return; } + // BIP155 defines feature negotiation of addrv2 and sendaddrv2, which must happen + // between VERSION and VERACK. if (msg_type == NetMsgType::SENDADDRV2) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send SENDADDRV2 message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a SENDADDRV2 message after VERACK. + LogPrint(BCLog::NET, "sendaddrv2 received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2595,6 +2861,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat s >> vAddr; if (!pfrom.RelayAddrsWithConn()) { + LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); return; } if (vAddr.size() > MAX_ADDR_TO_SEND) @@ -2638,8 +2905,10 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat m_connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; - if (pfrom.IsAddrFetchConn()) + if (pfrom.IsAddrFetchConn()) { + LogPrint(BCLog::NET, "addrfetch connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; + } return; } @@ -2693,7 +2962,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat } } else if (inv.IsGenTxMsg()) { const GenTxid gtxid = ToGenTxid(inv); - const bool fAlreadyHave = AlreadyHaveTx(gtxid, m_mempool); + const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); pfrom.AddKnownTx(inv.hash); @@ -2735,7 +3004,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat { LOCK(peer->m_getdata_requests_mutex); peer->m_getdata_requests.insert(peer->m_getdata_requests.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, *peer, m_chainparams, m_connman, m_mempool, interruptMsgProc); + ProcessGetData(pfrom, *peer, interruptMsgProc); } return; @@ -2766,7 +3035,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat a_recent_block = most_recent_block; } BlockValidationState state; - if (!ActivateBestChain(state, m_chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, m_chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -2774,7 +3043,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LOCK(cs_main); // Find the last block the caller has in the main chain - const CBlockIndex* pindex = FindForkInGlobalIndex(::ChainActive(), locator); + const CBlockIndex* pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); // Send the rest of the chain if (pindex) @@ -2827,7 +3096,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(req.blockhash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(req.blockhash); if (!pindex || !(pindex->nStatus & BLOCK_HAVE_DATA)) { LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom.GetId()); return; @@ -2881,7 +3150,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (locator.IsNull()) { // If locator is null, return the hashStop block - pindex = LookupBlockIndex(hashStop); + pindex = g_chainman.m_blockman.LookupBlockIndex(hashStop); if (!pindex) { return; } @@ -2894,7 +3163,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat else { // Find the last block the caller has in the main chain - pindex = FindForkInGlobalIndex(::ChainActive(), locator); + pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); if (pindex) pindex = ::ChainActive().Next(pindex); } @@ -2974,7 +3243,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid), m_mempool)) { + if (AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid))) { if (pfrom.HasPermission(PF_FORCERELAY)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing @@ -2989,10 +3258,10 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } - TxValidationState state; - std::list<CTransactionRef> lRemovedTxn; + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), m_mempool, ptx, false /* bypass_limits */); + const TxValidationState& state = result.m_state; - if (AcceptToMemoryPool(m_mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */)) { + if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { m_mempool.check(&::ChainstateActive().CoinsTip()); // As this version of the transaction was acceptable, we can forget about any // requests for it. @@ -3015,7 +3284,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat tx.GetHash().ToString(), m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); - for (const CTransactionRef& removedTx : lRemovedTxn) { + for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } @@ -3053,7 +3322,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // protocol for getting all unconfirmed parents. const GenTxid gtxid{/* is_wtxid=*/false, parent_txid}; pfrom.AddKnownTx(parent_txid); - if (!AlreadyHaveTx(gtxid, m_mempool)) AddTxAnnouncement(pfrom, gtxid, current_time); + if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } AddOrphanTx(ptx, pfrom.GetId()); @@ -3158,14 +3427,14 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat { LOCK(cs_main); - if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } - if (!LookupBlockIndex(cmpctblock.header.GetHash())) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.GetHash())) { received_new_header = true; } } @@ -3243,7 +3512,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; - if (!MarkBlockAsInFlight(m_mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&m_mempool)); else { @@ -3599,15 +3868,14 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat vRecv >> nonce; // Only process pong message if there is an outstanding ping (old ping without nonce should never pong) - if (pfrom.nPingNonceSent != 0) { - if (nonce == pfrom.nPingNonceSent) { + if (peer->m_ping_nonce_sent != 0) { + if (nonce == peer->m_ping_nonce_sent) { // Matching pong received, this ping is no longer outstanding bPingFinished = true; - const auto ping_time = ping_end - pfrom.m_ping_start.load(); + const auto ping_time = ping_end - peer->m_ping_start.load(); if (ping_time.count() >= 0) { - // Successful ping time measurement, replace previous - pfrom.nPingUsecTime = count_microseconds(ping_time); - pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), count_microseconds(ping_time)); + // Let connman know about this successful ping-pong + pfrom.PongReceived(ping_time); } else { // This should never happen sProblem = "Timing mishap"; @@ -3634,18 +3902,19 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", pfrom.GetId(), sProblem, - pfrom.nPingNonceSent, + peer->m_ping_nonce_sent, nonce, nAvail); } if (bPingFinished) { - pfrom.nPingNonceSent = 0; + peer->m_ping_nonce_sent = 0; } return; } if (msg_type == NetMsgType::FILTERLOAD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3668,6 +3937,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (msg_type == NetMsgType::FILTERADD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3695,6 +3965,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (msg_type == NetMsgType::FILTERCLEAR) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3712,7 +3983,6 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_feeFilter); pfrom.m_tx_relay->minFeeFilter = newFeeFilter; } LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); @@ -3756,49 +4026,46 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } -bool PeerManager::MaybeDiscourageAndDisconnect(CNode& pnode) +bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer) { - const NodeId peer_id{pnode.GetId()}; - PeerRef peer = GetPeerRef(peer_id); - if (peer == nullptr) return false; - { - LOCK(peer->m_misbehavior_mutex); + LOCK(peer.m_misbehavior_mutex); // There's nothing to do if the m_should_discourage flag isn't set - if (!peer->m_should_discourage) return false; + if (!peer.m_should_discourage) return false; - peer->m_should_discourage = false; + peer.m_should_discourage = false; } // peer.m_misbehavior_mutex if (pnode.HasPermission(PF_NOBAN)) { // We never disconnect or discourage peers for bad behavior if they have the NOBAN permission flag - LogPrintf("Warning: not punishing noban peer %d!\n", peer_id); + LogPrintf("Warning: not punishing noban peer %d!\n", peer.m_id); return false; } if (pnode.IsManualConn()) { // We never disconnect or discourage manual peers for bad behavior - LogPrintf("Warning: not punishing manually connected peer %d!\n", peer_id); + LogPrintf("Warning: not punishing manually connected peer %d!\n", peer.m_id); return false; } if (pnode.addr.IsLocal()) { // We disconnect local peers for bad behavior but don't discourage (since that would discourage // all peers on the same local address) - LogPrintf("Warning: disconnecting but not discouraging local peer %d!\n", peer_id); + LogPrint(BCLog::NET, "Warning: disconnecting but not discouraging %s peer %d!\n", + pnode.m_inbound_onion ? "inbound onion" : "local", peer.m_id); pnode.fDisconnect = true; return true; } // Normal case: Disconnect the peer and discourage all nodes sharing the address - LogPrint(BCLog::NET, "Disconnecting and discouraging peer %d!\n", peer_id); + LogPrint(BCLog::NET, "Disconnecting and discouraging peer %d!\n", peer.m_id); if (m_banman) m_banman->Discourage(pnode.addr); m_connman.DisconnectNode(pnode.addr); return true; } -bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) +bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) { bool fMoreWork = false; @@ -3808,7 +4075,7 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgP { LOCK(peer->m_getdata_requests_mutex); if (!peer->m_getdata_requests.empty()) { - ProcessGetData(*pfrom, *peer, m_chainparams, m_connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, *peer, interruptMsgProc); } } @@ -3835,14 +4102,12 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgP } // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + if (pfrom->fPauseSend) return false; std::list<CNetMessage> msgs; { LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; + if (pfrom->vProcessMsg.empty()) return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; @@ -3851,6 +4116,10 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgP } CNetMessage& msg(msgs.front()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + } + msg.SetVersion(pfrom->GetCommonVersion()); const std::string& msg_type = msg.m_command; @@ -3873,7 +4142,7 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgP return fMoreWork; } -void PeerManager::ConsiderEviction(CNode& pto, int64_t time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) { AssertLockHeld(cs_main); @@ -3926,7 +4195,7 @@ void PeerManager::ConsiderEviction(CNode& pto, int64_t time_in_seconds) } } -void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) +void PeerManagerImpl::EvictExtraOutboundPeers(int64_t time_in_seconds) { // If we have any extra block-relay-only peers, disconnect the youngest unless // it's given us a block -- in which case, compare with the second-youngest, and @@ -4027,7 +4296,7 @@ void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) } } -void PeerManager::CheckForStaleTipAndEvictPeers() +void PeerManagerImpl::CheckForStaleTipAndEvictPeers() { LOCK(cs_main); @@ -4038,8 +4307,8 @@ void PeerManager::CheckForStaleTipAndEvictPeers() if (time_in_seconds > m_stale_tip_check_time) { // Check whether our tip is stale, and if so, allow using an extra // outbound peer - if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale(m_chainparams.GetConsensus())) { - LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); + if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale()) { + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - m_last_tip_update); m_connman.SetTryNewOutboundPeer(true); } else if (m_connman.GetTryNewOutboundPeer()) { m_connman.SetTryNewOutboundPeer(false); @@ -4053,6 +4322,50 @@ void PeerManager::CheckForStaleTipAndEvictPeers() } } +void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer) +{ + // Use mockable time for ping timeouts. + // This means that setmocktime may cause pings to time out. + auto now = GetTime<std::chrono::microseconds>(); + + if (m_connman.RunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { + LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); + node_to.fDisconnect = true; + return; + } + + const CNetMsgMaker msgMaker(node_to.GetCommonVersion()); + bool pingSend = false; + + if (peer.m_ping_queued) { + // RPC ping request by user + pingSend = true; + } + + if (peer.m_ping_nonce_sent == 0 && now > peer.m_ping_start.load() + PING_INTERVAL) { + // Ping automatically sent as a latency probe & keepalive. + pingSend = true; + } + + if (pingSend) { + uint64_t nonce = 0; + while (nonce == 0) { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + peer.m_ping_queued = false; + peer.m_ping_start = now; + if (node_to.GetCommonVersion() > BIP0031_VERSION) { + peer.m_ping_nonce_sent = nonce; + m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING, nonce)); + } else { + // Peer is too old to support ping command with nonce, pong will never arrive. + peer.m_ping_nonce_sent = 0; + m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING)); + } + } +} + namespace { class CompareInvMempoolOrder { @@ -4074,14 +4387,15 @@ public: }; } -bool PeerManager::SendMessages(CNode* pto) +bool PeerManagerImpl::SendMessages(CNode* pto) { PeerRef peer = GetPeerRef(pto->GetId()); + if (!peer) return false; const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll // disconnect misbehaving peers even before the version handshake is complete. - if (MaybeDiscourageAndDisconnect(*pto)) return true; + if (MaybeDiscourageAndDisconnect(*pto, *peer)) return true; // Don't send anything until the version handshake is complete if (!pto->fSuccessfullyConnected || pto->fDisconnect) @@ -4090,34 +4404,10 @@ bool PeerManager::SendMessages(CNode* pto) // If we get here, the outgoing message serialization version is set and can't change. const CNetMsgMaker msgMaker(pto->GetCommonVersion()); - // - // Message: ping - // - bool pingSend = false; - if (pto->fPingQueued) { - // RPC ping request by user - pingSend = true; - } - if (pto->nPingNonceSent == 0 && pto->m_ping_start.load() + PING_INTERVAL < GetTime<std::chrono::microseconds>()) { - // Ping automatically sent as a latency probe & keepalive. - pingSend = true; - } - if (pingSend) { - uint64_t nonce = 0; - while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - pto->fPingQueued = false; - pto->m_ping_start = GetTime<std::chrono::microseconds>(); - if (pto->GetCommonVersion() > BIP0031_VERSION) { - pto->nPingNonceSent = nonce; - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); - } else { - // Peer is too old to support ping command with nonce, pong will never arrive. - pto->nPingNonceSent = 0; - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); - } - } + MaybeSendPing(*pto, *peer); + + // MaybeSendPing may have marked peer for disconnection + if (pto->fDisconnect) return true; { LOCK(cs_main); @@ -4127,7 +4417,9 @@ bool PeerManager::SendMessages(CNode* pto) // Address refresh broadcast auto current_time = GetTime<std::chrono::microseconds>(); - if (pto->RelayAddrsWithConn() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { + if (fListen && pto->RelayAddrsWithConn() && + !::ChainstateActive().IsInitialBlockDownload() && + pto->m_next_local_addr_send < current_time) { // If we've sent before, clear the bloom filter for the peer, so that our // self-announcement will actually go out. // This might be unnecessary if the bloom filter has already rolled @@ -4137,7 +4429,10 @@ bool PeerManager::SendMessages(CNode* pto) if (pto->m_next_local_addr_send != 0us) { pto->m_addr_known->reset(); } - AdvertiseLocal(pto); + if (Optional<CAddress> local_addr = GetLocalAddrForPeer(pto)) { + FastRandomContext insecure_rand; + pto->PushAddress(*local_addr, insecure_rand); + } pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } @@ -4232,7 +4527,7 @@ bool PeerManager::SendMessages(CNode* pto) // then send all headers past that one. If we come across any // headers that aren't on ::ChainActive(), give up. for (const uint256& hash : peer->m_blocks_for_headers_relay) { - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); assert(pindex); if (::ChainActive()[pindex->nHeight] != pindex) { // Bail out if we reorged away from this block @@ -4324,7 +4619,7 @@ bool PeerManager::SendMessages(CNode* pto) // in the past. if (!peer->m_blocks_for_headers_relay.empty()) { const uint256& hashToAnnounce = peer->m_blocks_for_headers_relay.back(); - const CBlockIndex* pindex = LookupBlockIndex(hashToAnnounce); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashToAnnounce); assert(pindex); // Warn if we're announcing a block that is not on the main chain. @@ -4388,11 +4683,7 @@ bool PeerManager::SendMessages(CNode* pto) if (fSendTrickle && pto->m_tx_relay->fSendMempool) { auto vtxinfo = m_mempool.infoAll(); pto->m_tx_relay->fSendMempool = false; - CFeeRate filterrate; - { - LOCK(pto->m_tx_relay->cs_feeFilter); - filterrate = CFeeRate(pto->m_tx_relay->minFeeFilter); - } + const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; LOCK(pto->m_tx_relay->cs_filter); @@ -4426,11 +4717,7 @@ bool PeerManager::SendMessages(CNode* pto) for (std::set<uint256>::iterator it = pto->m_tx_relay->setInventoryTxToSend.begin(); it != pto->m_tx_relay->setInventoryTxToSend.end(); it++) { vInvTx.push_back(it); } - CFeeRate filterrate; - { - LOCK(pto->m_tx_relay->cs_feeFilter); - filterrate = CFeeRate(pto->m_tx_relay->minFeeFilter); - } + const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); @@ -4574,11 +4861,11 @@ bool PeerManager::SendMessages(CNode* pto) if (!pto->fClient && ((fFetch && !pto->m_limited_node) || !::ChainstateActive().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); + FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(*pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(m_mempool, pto->GetId(), pindex->GetBlockHash(), pindex); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->GetId()); } @@ -4600,7 +4887,7 @@ bool PeerManager::SendMessages(CNode* pto) entry.second.GetHash().ToString(), entry.first); } for (const GenTxid& gtxid : requestable) { - if (!AlreadyHaveTx(gtxid, m_mempool)) { + if (!AlreadyHaveTx(gtxid)) { LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", gtxid.GetHash().ToString(), pto->GetId()); vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), gtxid.GetHash()); diff --git a/src/net_processing.h b/src/net_processing.h index 322fabbe11..d7be453df5 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -6,19 +6,13 @@ #ifndef BITCOIN_NET_PROCESSING_H #define BITCOIN_NET_PROCESSING_H -#include <consensus/params.h> #include <net.h> #include <sync.h> -#include <txrequest.h> #include <validationinterface.h> -class BlockTransactionsRequest; -class BlockValidationState; -class CBlockHeader; class CChainParams; class CTxMemPool; class ChainstateManager; -class TxValidationState; extern RecursiveMutex cs_main; extern RecursiveMutex g_cs_orphans; @@ -36,219 +30,46 @@ struct CNodeStateStats { int nSyncHeight = -1; int nCommonHeight = -1; int m_starting_height = -1; + int64_t m_ping_wait_usec; std::vector<int> vHeightInFlight; }; -/** - * Data structure for an individual peer. This struct is not protected by - * cs_main since it does not contain validation-critical data. - * - * Memory is owned by shared pointers and this object is destructed when - * the refcount drops to zero. - * - * Mutexes inside this struct must not be held when locking m_peer_mutex. - * - * TODO: move most members from CNodeState to this structure. - * TODO: move remaining application-layer data members from CNode to this structure. - */ -struct Peer { - /** Same id as the CNode object for this peer */ - const NodeId m_id{0}; - - /** Protects misbehavior data members */ - Mutex m_misbehavior_mutex; - /** Accumulated misbehavior score for this peer */ - int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; - /** Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). */ - bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; - - /** Protects block inventory data members */ - Mutex m_block_inv_mutex; - /** List of blocks that we'll announce via an `inv` message. - * There is no final sorting before sending, as they are always sent - * immediately and in the order requested. */ - std::vector<uint256> m_blocks_for_inv_relay GUARDED_BY(m_block_inv_mutex); - /** Unfiltered list of blocks that we'd like to announce via a `headers` - * message. If we can't announce via a `headers` message, we'll fall back to - * announcing via `inv`. */ - std::vector<uint256> m_blocks_for_headers_relay GUARDED_BY(m_block_inv_mutex); - /** The final block hash that we sent in an `inv` message to this peer. - * When the peer requests this block, we send an `inv` message to trigger - * the peer to request the next sequence of block hashes. - * Most peers use headers-first syncing, which doesn't use this mechanism */ - uint256 m_continuation_block GUARDED_BY(m_block_inv_mutex) {}; - - /** This peer's reported block height when we connected */ - std::atomic<int> m_starting_height{-1}; - - /** Set of txids to reconsider once their parent transactions have been accepted **/ - std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); - - /** Protects m_getdata_requests **/ - Mutex m_getdata_requests_mutex; - /** Work queue of items requested by this peer **/ - std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - - explicit Peer(NodeId id) : m_id(id) {} -}; - -using PeerRef = std::shared_ptr<Peer>; - -class PeerManager final : public CValidationInterface, public NetEventsInterface { +class PeerManager : public CValidationInterface, public NetEventsInterface +{ public: - PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, - CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, - bool ignore_incoming_txs); + static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs); + virtual ~PeerManager() { } - /** - * Overridden from CValidationInterface. - */ - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; - void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; - /** - * Overridden from CValidationInterface. - */ - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; - /** - * Overridden from CValidationInterface. - */ - void BlockChecked(const CBlock& block, const BlockValidationState& state) override; - /** - * Overridden from CValidationInterface. - */ - void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; + /** Get statistics from node state */ + virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) = 0; - /** Initialize a peer by adding it to mapNodeState and pushing a message requesting its version */ - void InitializeNode(CNode* pnode) override; - /** Handle removal of a peer by updating various state and removing it from mapNodeState */ - void FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) override; - /** - * Process protocol messages received from a given node - * - * @param[in] pfrom The node which we have received messages from. - * @param[in] interrupt Interrupt condition for processing threads - */ - bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override; - /** - * Send queued protocol messages to be sent to a give node. - * - * @param[in] pto The node which we are sending messages to. - * @return True if there is more work to be done - */ - bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); + /** Whether this node ignores txs received over p2p. */ + virtual bool IgnoresIncomingTxs() = 0; - /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound */ - void CheckForStaleTipAndEvictPeers(); - /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ - void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ - void ReattemptInitialBroadcast(CScheduler& scheduler) const; + /** Send ping message to all peers */ + virtual void SendPings() = 0; - /** Process a single message from a peer. Public for fuzz testing */ - void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc); + /** Set the best height */ + virtual void SetBestHeight(int height) = 0; /** * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. * Public for unit testing. */ - void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message); - - /** Get statistics from node state */ - bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats); - - /** Set the best height */ - void SetBestHeight(int height) { m_best_height = height; }; - - /** Whether this node ignores txs received over p2p. */ - bool IgnoresIncomingTxs() { return m_ignore_incoming_txs; }; - -private: - /** Get a shared pointer to the Peer object. - * May return an empty shared_ptr if the Peer object can't be found. */ - PeerRef GetPeerRef(NodeId id) const; - - /** Get a shared pointer to the Peer object and remove it from m_peer_map. - * May return an empty shared_ptr if the Peer object can't be found. */ - PeerRef RemovePeer(NodeId id); + virtual void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) = 0; /** - * Potentially mark a node discouraged based on the contents of a BlockValidationState object - * - * @param[in] via_compact_block this bool is passed in because net_processing should - * punish peers differently depending on whether the data was provided in a compact - * block message or not. If the compact block had a valid header, but contained invalid - * txs, the peer should not be punished. See BIP 152. - * - * @return Returns true if the peer was punished (probably disconnected) - */ - bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, - bool via_compact_block, const std::string& message = ""); - - /** - * Potentially disconnect and discourage a node based on the contents of a TxValidationState object - * - * @return Returns true if the peer was punished (probably disconnected) - */ - bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = ""); - - /** Maybe disconnect a peer and discourage future connections from its address. - * - * @param[in] pnode The node to check. - * @return True if the peer was marked for disconnection in this function + * Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound. + * Public for unit testing. */ - bool MaybeDiscourageAndDisconnect(CNode& pnode); + virtual void CheckForStaleTipAndEvictPeers() = 0; - void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); - /** Process a single headers message from a peer. */ - void ProcessHeadersMessage(CNode& pfrom, const Peer& peer, - const std::vector<CBlockHeader>& headers, - bool via_compact_block); - - void SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req); - - /** Register with TxRequestTracker that an INV has been received from a - * peer. The announcement parameters are decided in PeerManager and then - * passed to TxRequestTracker. */ - void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - - /** Send a version message to a peer */ - void PushNodeVersion(CNode& pnode, int64_t nTime); - - const CChainParams& m_chainparams; - CConnman& m_connman; - /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ - BanMan* const m_banman; - ChainstateManager& m_chainman; - CTxMemPool& m_mempool; - TxRequestTracker m_txrequest GUARDED_BY(::cs_main); - - /** The height of the best chain */ - std::atomic<int> m_best_height{-1}; - - int64_t m_stale_tip_check_time; //!< Next time to check for stale tip - - /** Whether this node is running in blocks only mode */ - const bool m_ignore_incoming_txs; - - /** Whether we've completed initial sync yet, for determining when to turn - * on extra block-relay-only peers. */ - bool m_initial_sync_finished{false}; - - /** Protects m_peer_map. This mutex must not be locked while holding a lock - * on any of the mutexes inside a Peer object. */ - mutable Mutex m_peer_mutex; - /** - * Map of all Peer objects, keyed by peer id. This map is protected - * by the m_peer_mutex. Once a shared pointer reference is - * taken, the lock may be released. Individual fields are protected by - * their own locks. - */ - std::map<NodeId, PeerRef> m_peer_map GUARDED_BY(m_peer_mutex); + /** Process a single message from a peer. Public for fuzz testing */ + virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) = 0; }; /** Relay transaction to every node */ diff --git a/src/netaddress.cpp b/src/netaddress.cpp index b1f9d32d34..69edc15c66 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -221,25 +221,34 @@ static void Checksum(Span<const uint8_t> addr_pubkey, uint8_t (&checksum)[CHECKS }; // namespace torv3 -/** - * Parse a TOR address and set this object to it. - * - * @returns Whether or not the operation was successful. - * - * @see CNetAddr::IsTor() - */ -bool CNetAddr::SetSpecial(const std::string& str) +bool CNetAddr::SetSpecial(const std::string& addr) +{ + if (!ValidAsCString(addr)) { + return false; + } + + if (SetTor(addr)) { + return true; + } + + if (SetI2P(addr)) { + return true; + } + + return false; +} + +bool CNetAddr::SetTor(const std::string& addr) { static const char* suffix{".onion"}; static constexpr size_t suffix_len{6}; - if (!ValidAsCString(str) || str.size() <= suffix_len || - str.substr(str.size() - suffix_len) != suffix) { + if (addr.size() <= suffix_len || addr.substr(addr.size() - suffix_len) != suffix) { return false; } bool invalid; - const auto& input = DecodeBase32(str.substr(0, str.size() - suffix_len).c_str(), &invalid); + const auto& input = DecodeBase32(addr.substr(0, addr.size() - suffix_len).c_str(), &invalid); if (invalid) { return false; @@ -275,6 +284,34 @@ bool CNetAddr::SetSpecial(const std::string& str) return false; } +bool CNetAddr::SetI2P(const std::string& addr) +{ + // I2P addresses that we support consist of 52 base32 characters + ".b32.i2p". + static constexpr size_t b32_len{52}; + static const char* suffix{".b32.i2p"}; + static constexpr size_t suffix_len{8}; + + if (addr.size() != b32_len + suffix_len || ToLower(addr.substr(b32_len)) != suffix) { + return false; + } + + // Remove the ".b32.i2p" suffix and pad to a multiple of 8 chars, so DecodeBase32() + // can decode it. + const std::string b32_padded = addr.substr(0, b32_len) + "===="; + + bool invalid; + const auto& address_bytes = DecodeBase32(b32_padded.c_str(), &invalid); + + if (invalid || address_bytes.size() != ADDR_I2P_SIZE) { + return false; + } + + m_net = NET_I2P; + m_addr.assign(address_bytes.begin(), address_bytes.end()); + + return true; +} + CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) { m_net = NET_IPV4; @@ -841,6 +878,11 @@ int CNetAddr::GetReachabilityFrom(const CNetAddr *paddrPartner) const case NET_IPV4: return REACH_IPV4; // Tor users can connect to IPv4 as well case NET_ONION: return REACH_PRIVATE; } + case NET_I2P: + switch (ourNet) { + case NET_I2P: return REACH_PRIVATE; + default: return REACH_DEFAULT; + } case NET_TEREDO: switch(ourNet) { default: return REACH_DEFAULT; @@ -1068,15 +1110,24 @@ CSubNet::CSubNet(const CNetAddr& addr, const CNetAddr& mask) : CSubNet() CSubNet::CSubNet(const CNetAddr& addr) : CSubNet() { - valid = addr.IsIPv4() || addr.IsIPv6(); - if (!valid) { + switch (addr.m_net) { + case NET_IPV4: + case NET_IPV6: + valid = true; + assert(addr.m_addr.size() <= sizeof(netmask)); + memset(netmask, 0xFF, addr.m_addr.size()); + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + valid = true; + break; + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: return; } - assert(addr.m_addr.size() <= sizeof(netmask)); - - memset(netmask, 0xFF, addr.m_addr.size()); - network = addr; } @@ -1088,6 +1139,21 @@ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; + + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + case NET_INTERNAL: + return addr == network; + case NET_UNROUTABLE: + case NET_MAX: + return false; + } + assert(network.m_addr.size() == addr.m_addr.size()); for (size_t x = 0; x < addr.m_addr.size(); ++x) { if ((addr.m_addr[x] & netmask[x]) != network.m_addr[x]) { @@ -1099,18 +1165,35 @@ bool CSubNet::Match(const CNetAddr &addr) const std::string CSubNet::ToString() const { - assert(network.m_addr.size() <= sizeof(netmask)); + std::string suffix; - uint8_t cidr = 0; + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: { + assert(network.m_addr.size() <= sizeof(netmask)); - for (size_t i = 0; i < network.m_addr.size(); ++i) { - if (netmask[i] == 0x00) { - break; + uint8_t cidr = 0; + + for (size_t i = 0; i < network.m_addr.size(); ++i) { + if (netmask[i] == 0x00) { + break; + } + cidr += NetmaskBits(netmask[i]); } - cidr += NetmaskBits(netmask[i]); + + suffix = strprintf("/%u", cidr); + break; + } + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: + break; } - return network.ToString() + strprintf("/%u", cidr); + return network.ToString() + suffix; } bool CSubNet::IsValid() const @@ -1120,7 +1203,19 @@ bool CSubNet::IsValid() const bool CSubNet::SanityCheck() const { - if (!(network.IsIPv4() || network.IsIPv6())) return false; + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + return true; + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: + return false; + } for (size_t x = 0; x < network.m_addr.size(); ++x) { if (network.m_addr[x] & ~netmask[x]) return false; diff --git a/src/netaddress.h b/src/netaddress.h index cf878fe374..897ce46cda 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -36,7 +36,7 @@ static constexpr int ADDRV2_FORMAT = 0x20000000; * @note An address may belong to more than one network, for example `10.0.0.1` * belongs to both `NET_UNROUTABLE` and `NET_IPV4`. * Keep these sequential starting from 0 and `NET_MAX` as the last entry. - * We have loops like `for (int i = 0; i < NET_MAX; i++)` that expect to iterate + * We have loops like `for (int i = 0; i < NET_MAX; ++i)` that expect to iterate * over all enum values and also `GetExtNetwork()` "extends" this enum by * introducing standalone constants starting from `NET_MAX`. */ @@ -151,7 +151,16 @@ class CNetAddr bool SetInternal(const std::string& name); - bool SetSpecial(const std::string &strName); // for Tor addresses + /** + * Parse a Tor or I2P address and set this object to it. + * @param[in] addr Address to parse, for example + * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion or + * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. + * @returns Whether the operation was successful. + * @see CNetAddr::IsTor(), CNetAddr::IsI2P() + */ + bool SetSpecial(const std::string& addr); + bool IsBindAny() const; // INADDR_ANY equivalent bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) @@ -249,6 +258,25 @@ class CNetAddr private: /** + * Parse a Tor address and set this object to it. + * @param[in] addr Address to parse, must be a valid C string, for example + * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion or + * 6hzph5hv6337r6p2.onion. + * @returns Whether the operation was successful. + * @see CNetAddr::IsTor() + */ + bool SetTor(const std::string& addr); + + /** + * Parse an I2P address and set this object to it. + * @param[in] addr Address to parse, must be a valid C string, for example + * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. + * @returns Whether the operation was successful. + * @see CNetAddr::IsI2P() + */ + bool SetI2P(const std::string& addr); + + /** * BIP155 network ids recognized by this software. */ enum BIP155Network : uint8_t { @@ -462,11 +490,33 @@ class CSubNet bool SanityCheck() const; public: + /** + * Construct an invalid subnet (empty, `Match()` always returns false). + */ CSubNet(); + + /** + * Construct from a given network start and number of bits (CIDR mask). + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask CIDR mask, must be in [0, 32] for IPv4 addresses and in [0, 128] for + * IPv6 addresses. Otherwise an invalid subnet is created. + */ CSubNet(const CNetAddr& addr, uint8_t mask); + + /** + * Construct from a given network start and mask. + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask Network mask, must be of the same type as `addr` and not contain 0-bits + * followed by 1-bits. Otherwise an invalid subnet is created. + */ CSubNet(const CNetAddr& addr, const CNetAddr& mask); - //constructor for single ip subnet (<ipv4>/32 or <ipv6>/128) + /** + * Construct a single-host subnet. + * @param[in] addr The sole address to be contained in the subnet, can also be non-IPv[46]. + */ explicit CSubNet(const CNetAddr& addr); bool Match(const CNetAddr &addr) const; diff --git a/src/netbase.cpp b/src/netbase.cpp index 264029d8a2..7dc616080d 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -5,15 +5,21 @@ #include <netbase.h> +#include <compat.h> #include <sync.h> #include <tinyformat.h> +#include <util/sock.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/time.h> #include <atomic> +#include <chrono> #include <cstdint> +#include <functional> #include <limits> +#include <memory> #ifndef WIN32 #include <fcntl.h> @@ -25,10 +31,6 @@ #include <poll.h> #endif -#if !defined(MSG_NOSIGNAL) -#define MSG_NOSIGNAL 0 -#endif - // Settings static Mutex g_proxyinfo_mutex; static proxyType proxyInfo[NET_MAX] GUARDED_BY(g_proxyinfo_mutex); @@ -49,13 +51,16 @@ enum Network ParseNetwork(const std::string& net_in) { LogPrintf("Warning: net name 'tor' is deprecated and will be removed in the future. You should use 'onion' instead.\n"); return NET_ONION; } + if (net == "i2p") { + return NET_I2P; + } return NET_UNROUTABLE; } std::string GetNetworkName(enum Network net) { switch (net) { - case NET_UNROUTABLE: return "unroutable"; + case NET_UNROUTABLE: return "not_publicly_routable"; case NET_IPV4: return "ipv4"; case NET_IPV6: return "ipv6"; case NET_ONION: return "onion"; @@ -68,6 +73,20 @@ std::string GetNetworkName(enum Network net) assert(false); } +std::vector<std::string> GetNetworkNames(bool append_unroutable) +{ + std::vector<std::string> names; + for (int n = 0; n < NET_MAX; ++n) { + const enum Network network{static_cast<Network>(n)}; + if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; + names.emplace_back(GetNetworkName(network)); + } + if (append_unroutable) { + names.emplace_back(GetNetworkName(NET_UNROUTABLE)); + } + return names; +} + bool static LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { vIP.clear(); @@ -271,14 +290,6 @@ CService LookupNumeric(const std::string& name, int portDefault) return addr; } -struct timeval MillisToTimeval(int64_t nTimeout) -{ - struct timeval timeout; - timeout.tv_sec = nTimeout / 1000; - timeout.tv_usec = (nTimeout % 1000) * 1000; - return timeout; -} - /** SOCKS version */ enum SOCKSVersion: uint8_t { SOCKS4 = 0x04, @@ -336,8 +347,7 @@ enum class IntrRecvError { * @param data The buffer where the read bytes should be stored. * @param len The number of bytes to read into the specified buffer. * @param timeout The total timeout in milliseconds for this read. - * @param hSocket The socket (has to be in non-blocking mode) from which to read - * bytes. + * @param sock The socket (has to be in non-blocking mode) from which to read bytes. * * @returns An IntrRecvError indicating the resulting status of this read. * IntrRecvError::OK only if all of the specified number of bytes were @@ -347,15 +357,12 @@ enum class IntrRecvError { * Sockets can be made non-blocking with SetSocketNonBlocking(const * SOCKET&, bool). */ -static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const SOCKET& hSocket) +static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { int64_t curTime = GetTimeMillis(); int64_t endTime = curTime + timeout; - // Maximum time to wait for I/O readiness. It will take up until this time - // (in millis) to break off in case of an interruption. - const int64_t maxWait = 1000; while (len > 0 && curTime < endTime) { - ssize_t ret = recv(hSocket, (char*)data, len, 0); // Optimistically try the recv first + ssize_t ret = sock.Recv(data, len, 0); // Optimistically try the recv first if (ret > 0) { len -= ret; data += ret; @@ -364,25 +371,11 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c } else { // Other error or blocking int nErr = WSAGetLastError(); if (nErr == WSAEINPROGRESS || nErr == WSAEWOULDBLOCK || nErr == WSAEINVAL) { - if (!IsSelectableSocket(hSocket)) { - return IntrRecvError::NetworkError; - } - // Only wait at most maxWait milliseconds at a time, unless + // Only wait at most MAX_WAIT_FOR_IO at a time, unless // we're approaching the end of the specified total timeout - int timeout_ms = std::min(endTime - curTime, maxWait); -#ifdef USE_POLL - struct pollfd pollfd = {}; - pollfd.fd = hSocket; - pollfd.events = POLLIN; - int nRet = poll(&pollfd, 1, timeout_ms); -#else - struct timeval tval = MillisToTimeval(timeout_ms); - fd_set fdset; - FD_ZERO(&fdset); - FD_SET(hSocket, &fdset); - int nRet = select(hSocket + 1, &fdset, nullptr, nullptr, &tval); -#endif - if (nRet == SOCKET_ERROR) { + const auto remaining = std::chrono::milliseconds{endTime - curTime}; + const auto timeout = std::min(remaining, std::chrono::milliseconds{MAX_WAIT_FOR_IO}); + if (!sock.Wait(timeout, Sock::RECV)) { return IntrRecvError::NetworkError; } } else { @@ -436,7 +429,7 @@ static std::string Socks5ErrorString(uint8_t err) * @param port The destination port. * @param auth The credentials with which to authenticate with the specified * SOCKS5 proxy. - * @param hSocket The SOCKS5 proxy socket. + * @param sock The SOCKS5 proxy socket. * * @returns Whether or not the operation succeeded. * @@ -446,7 +439,7 @@ static std::string Socks5ErrorString(uint8_t err) * @see <a href="https://www.ietf.org/rfc/rfc1928.txt">RFC1928: SOCKS Protocol * Version 5</a> */ -static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, const SOCKET& hSocket) +static bool Socks5(const std::string& strDest, int port, const ProxyCredentials* auth, const Sock& sock) { IntrRecvError recvr; LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); @@ -464,12 +457,12 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5Init.push_back(0x01); // 1 method identifier follows... vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); + ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { return error("Error sending to proxy"); } uint8_t pchRet1[2]; - if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; } @@ -486,13 +479,13 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); vAuth.push_back(auth->password.size()); vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); + ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { return error("Error sending authentication to proxy"); } LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); uint8_t pchRetA[2]; - if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { return error("Error reading proxy authentication response"); } if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { @@ -512,12 +505,12 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); vSocks5.push_back((port >> 8) & 0xFF); vSocks5.push_back((port >> 0) & 0xFF); - ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); + ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { return error("Error sending to proxy"); } uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { if (recvr == IntrRecvError::Timeout) { /* If a timeout happens here, this effectively means we timed out while connecting * to the remote node. This is very common for Tor, so do not print an @@ -541,16 +534,16 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials uint8_t pchRet3[256]; switch (pchRet2[3]) { - case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, SOCKS5_RECV_TIMEOUT, hSocket); break; - case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, SOCKS5_RECV_TIMEOUT, hSocket); break; + case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, SOCKS5_RECV_TIMEOUT, sock); break; + case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, SOCKS5_RECV_TIMEOUT, sock); break; case SOCKS5Atyp::DOMAINNAME: { - recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, hSocket); + recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); } int nRecv = pchRet3[0]; - recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, hSocket); + recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, sock); break; } default: return error("Error: malformed proxy response"); @@ -558,41 +551,35 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); } - if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { return error("Error reading from proxy"); } LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } -/** - * Try to create a socket file descriptor with specific properties in the - * communications domain (address family) of the specified service. - * - * For details on the desired properties, see the inline comments in the source - * code. - */ -SOCKET CreateSocket(const CService &addrConnect) +std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) { // Create a sockaddr from the specified service. struct sockaddr_storage sockaddr; socklen_t len = sizeof(sockaddr); - if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot create socket for %s: unsupported network\n", addrConnect.ToString()); - return INVALID_SOCKET; + if (!address_family.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { + LogPrintf("Cannot create socket for %s: unsupported network\n", address_family.ToString()); + return nullptr; } // Create a TCP socket in the address family of the specified service. SOCKET hSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); - if (hSocket == INVALID_SOCKET) - return INVALID_SOCKET; + if (hSocket == INVALID_SOCKET) { + return nullptr; + } // Ensure that waiting for I/O on this socket won't result in undefined // behavior. if (!IsSelectableSocket(hSocket)) { CloseSocket(hSocket); LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); - return INVALID_SOCKET; + return nullptr; } #ifdef SO_NOSIGPIPE @@ -608,11 +595,14 @@ SOCKET CreateSocket(const CService &addrConnect) // Set the non-blocking option on the socket. if (!SetSocketNonBlocking(hSocket, true)) { CloseSocket(hSocket); - LogPrintf("CreateSocket: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); + LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); + return nullptr; } - return hSocket; + return std::make_unique<Sock>(hSocket); } +std::function<std::unique_ptr<Sock>(const CService&)> CreateSock = CreateSockTCP; + template<typename... Args> static void LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args) { std::string error_message = tfm::format(fmt, args...); @@ -786,7 +776,7 @@ bool IsProxy(const CNetAddr &addr) { * @param proxy The SOCKS5 proxy. * @param strDest The destination service to which to connect. * @param port The destination port. - * @param hSocket The socket on which to connect to the SOCKS5 proxy. + * @param sock The socket on which to connect to the SOCKS5 proxy. * @param nTimeout Wait this many milliseconds for the connection to the SOCKS5 * proxy to be established. * @param[out] outProxyConnectionFailed Whether or not the connection to the @@ -794,10 +784,10 @@ bool IsProxy(const CNetAddr &addr) { * * @returns Whether or not the operation succeeded. */ -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool& outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType& proxy, const std::string& strDest, int port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed) { // first connect to proxy server - if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout, true)) { + if (!ConnectSocketDirectly(proxy.proxy, sock.Get(), nTimeout, true)) { outProxyConnectionFailed = true; return false; } @@ -806,11 +796,11 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (uint16_t)port, &random_auth, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, &random_auth, sock)) { return false; } } else { - if (!Socks5(strDest, (uint16_t)port, 0, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, 0, sock)) { return false; } } @@ -869,57 +859,6 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) return false; } -#ifdef WIN32 -std::string NetworkErrorString(int err) -{ - wchar_t buf[256]; - buf[0] = 0; - if(FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, - nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - buf, ARRAYSIZE(buf), nullptr)) - { - return strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err); - } - else - { - return strprintf("Unknown error (%d)", err); - } -} -#else -std::string NetworkErrorString(int err) -{ - char buf[256]; - buf[0] = 0; - /* Too bad there are two incompatible implementations of the - * thread-safe strerror. */ - const char *s; -#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ - s = strerror_r(err, buf, sizeof(buf)); -#else /* POSIX variant always returns message in buffer */ - s = buf; - if (strerror_r(err, buf, sizeof(buf))) - buf[0] = 0; -#endif - return strprintf("%s (%d)", s, err); -} -#endif - -bool CloseSocket(SOCKET& hSocket) -{ - if (hSocket == INVALID_SOCKET) - return false; -#ifdef WIN32 - int ret = closesocket(hSocket); -#else - int ret = close(hSocket); -#endif - if (ret) { - LogPrintf("Socket close failed: %d. Error: %s\n", hSocket, NetworkErrorString(WSAGetLastError())); - } - hSocket = INVALID_SOCKET; - return ret != SOCKET_ERROR; -} - bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking) { if (fNonBlocking) { diff --git a/src/netbase.h b/src/netbase.h index ac4cd97673..847a72ca8e 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -12,7 +12,10 @@ #include <compat.h> #include <netaddress.h> #include <serialize.h> +#include <util/sock.h> +#include <functional> +#include <memory> #include <stdint.h> #include <string> #include <vector> @@ -39,6 +42,8 @@ public: enum Network ParseNetwork(const std::string& net); std::string GetNetworkName(enum Network net); +/** Return a vector of publicly routable Network names; optionally append NET_UNROUTABLE. */ +std::vector<std::string> GetNetworkNames(bool append_unroutable = false); bool SetProxy(enum Network net, const proxyType &addrProxy); bool GetProxy(enum Network net, proxyType &proxyInfoOut); bool IsProxy(const CNetAddr &addr); @@ -51,21 +56,25 @@ bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllo bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); CService LookupNumeric(const std::string& name, int portDefault = 0); bool LookupSubNet(const std::string& strSubnet, CSubNet& subnet); -SOCKET CreateSocket(const CService &addrConnect); + +/** + * Create a TCP socket in the given address family. + * @param[in] address_family The socket is created in the same address family as this address. + * @return pointer to the created Sock object or unique_ptr that owns nothing in case of failure + */ +std::unique_ptr<Sock> CreateSockTCP(const CService& address_family); + +/** + * Socket factory. Defaults to `CreateSockTCP()`, but can be overridden by unit tests. + */ +extern std::function<std::unique_ptr<Sock>(const CService&)> CreateSock; + bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout, bool manual_connection); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool& outProxyConnectionFailed); -/** Return readable error string for a network error code */ -std::string NetworkErrorString(int err); -/** Close socket and set hSocket to INVALID_SOCKET */ -bool CloseSocket(SOCKET& hSocket); +bool ConnectThroughProxy(const proxyType& proxy, const std::string& strDest, int port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking); /** Set the TCP_NODELAY flag on a socket */ bool SetSocketNoDelay(const SOCKET& hSocket); -/** - * Convert milliseconds to a struct timeval for e.g. select. - */ -struct timeval MillisToTimeval(int64_t nTimeout); void InterruptSocks5(bool interrupt); #endif // BITCOIN_NETBASE_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 02e50c4dbe..06fcc33725 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -6,6 +6,7 @@ #include <node/coinstats.h> #include <coins.h> +#include <crypto/muhash.h> #include <hash.h> #include <serialize.h> #include <uint256.h> @@ -24,31 +25,59 @@ static uint64_t GetBogoSize(const CScript& scriptPubKey) scriptPubKey.size() /* scriptPubKey */; } -static void ApplyStats(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) { - assert(!outputs.empty()); - ss << hash; - ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); - stats.nTransactions++; - for (const auto& output : outputs) { - ss << VARINT(output.first + 1); - ss << output.second.out.scriptPubKey; - ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + if (it == outputs.begin()) { + ss << hash; + ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); + } + + ss << VARINT(it->first + 1); + ss << it->second.out.scriptPubKey; + ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + + if (it == std::prev(outputs.end())) { + ss << VARINT(0u); } - ss << VARINT(0u); } -static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) {} + +static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) +{ + COutPoint outpoint = COutPoint(hash, it->first); + Coin coin = it->second; + + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + ss << outpoint; + ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + muhash.Insert(MakeUCharSpan(ss)); +} + +//! Warning: be very careful when changing this! assumeutxo and UTXO snapshot +//! validation commitments are reliant on the hash constructed by this +//! function. +//! +//! If the construction of this hash is changed, it will invalidate +//! existing UTXO snapshots. This will not result in any kind of consensus +//! failure, but it will force clients that were expecting to make use of +//! assumeutxo to do traditional IBD instead. +//! +//! It is also possible, though very unlikely, that a change in this +//! construction could cause a previously invalid (and potentially malicious) +//! UTXO snapshot to be considered valid. +template <typename T> +static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); stats.nTransactions++; - for (const auto& output : outputs) { + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + ApplyHash(stats, hash_obj, hash, outputs, it); + stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + stats.nTotalAmount += it->second.out.nValue; + stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); } } @@ -63,7 +92,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + stats.nHeight = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock)->nHeight; } PrepareHash(hash_obj, stats); @@ -104,6 +133,10 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_t CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); return GetUTXOStats(view, stats, ss, interruption_point); } + case(CoinStatsHashType::MUHASH): { + MuHash3072 muhash; + return GetUTXOStats(view, stats, muhash, interruption_point); + } case(CoinStatsHashType::NONE): { return GetUTXOStats(view, stats, nullptr, interruption_point); } @@ -116,10 +149,18 @@ static void PrepareHash(CHashWriter& ss, const CCoinsStats& stats) { ss << stats.hashBlock; } +// MuHash does not need the prepare step +static void PrepareHash(MuHash3072& muhash, CCoinsStats& stats) {} static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) { stats.hashSerialized = ss.GetHash(); } +static void FinalizeHash(MuHash3072& muhash, CCoinsStats& stats) +{ + uint256 out; + muhash.Finalize(out); + stats.hashSerialized = out; +} static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c56bfc2ad..f02b95235f 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -16,6 +16,7 @@ class CCoinsView; enum class CoinStatsHashType { HASH_SERIALIZED, + MUHASH, NONE, }; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index e07eaa33d8..ec976fe9bf 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -441,13 +441,13 @@ public: bool checkFinalTx(const CTransaction& tx) override { LOCK(cs_main); - return CheckFinalTx(tx); + return CheckFinalTx(::ChainActive().Tip(), tx); } Optional<int> findLocatorFork(const CBlockLocator& locator) override { LOCK(cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (CBlockIndex* fork = FindForkInGlobalIndex(active, locator)) { + if (CBlockIndex* fork = g_chainman.m_blockman.FindForkInGlobalIndex(active, locator)) { return fork->nHeight; } return nullopt; @@ -456,7 +456,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - return FillBlock(LookupBlockIndex(hash), block, lock, active); + return FillBlock(g_chainman.m_blockman.LookupBlockIndex(hash), block, lock, active); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { @@ -468,7 +468,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { return FillBlock(ancestor, ancestor_out, lock, active); } @@ -479,8 +479,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block = LookupBlockIndex(block_hash); - const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = g_chainman.m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; return FillBlock(ancestor, ancestor_out, lock, active); } @@ -488,8 +488,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block1 = LookupBlockIndex(block_hash1); - const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* block1 = g_chainman.m_blockman.LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = g_chainman.m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. @@ -499,7 +499,7 @@ public: double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); - return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); + return GuessVerificationProgress(Params().TxData(), g_chainman.m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override { @@ -511,7 +511,7 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index d3bb9687a8..3b3fab7b6b 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -50,22 +50,22 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } if (!node.mempool->exists(hashTx)) { // Transaction is not already in the mempool. - TxValidationState state; if (max_tx_fee > 0) { // First, call ATMP with test_accept and check the fee. If ATMP // fails here, return error immediately. - CAmount fee{0}; - if (!AcceptToMemoryPool(*node.mempool, state, tx, - nullptr /* plTxnReplaced */, false /* bypass_limits */, /* test_accept */ true, &fee)) { - return HandleATMPError(state, err_string); - } else if (fee > max_tx_fee) { + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *node.mempool, tx, false /* bypass_limits */, + true /* test_accept */); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { + return HandleATMPError(result.m_state, err_string); + } else if (result.m_base_fees.value() > max_tx_fee) { return TransactionError::MAX_FEE_EXCEEDED; } } // Try to submit the transaction to the mempool. - if (!AcceptToMemoryPool(*node.mempool, state, tx, - nullptr /* plTxnReplaced */, false /* bypass_limits */)) { - return HandleATMPError(state, err_string); + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *node.mempool, tx, false /* bypass_limits */, + false /* test_accept */); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { + return HandleATMPError(result.m_state, err_string); } // Transaction was accepted to the mempool. diff --git a/src/outputtype.cpp b/src/outputtype.cpp index e978852826..d96fb282c5 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -19,8 +19,6 @@ static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; -const std::array<OutputType, 3> OUTPUT_TYPES = {OutputType::LEGACY, OutputType::P2SH_SEGWIT, OutputType::BECH32}; - bool ParseOutputType(const std::string& type, OutputType& output_type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { diff --git a/src/outputtype.h b/src/outputtype.h index 57316b92d6..88422e5824 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -20,7 +20,11 @@ enum class OutputType { BECH32, }; -extern const std::array<OutputType, 3> OUTPUT_TYPES; +static constexpr auto OUTPUT_TYPES = std::array{ + OutputType::LEGACY, + OutputType::P2SH_SEGWIT, + OutputType::BECH32, +}; [[nodiscard]] bool ParseOutputType(const std::string& str, OutputType& output_type); const std::string& FormatOutputType(OutputType type); diff --git a/src/policy/fees.h b/src/policy/fees.h index dd9f530c99..c444d71a31 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -11,6 +11,7 @@ #include <random.h> #include <sync.h> +#include <array> #include <map> #include <memory> #include <string> @@ -25,9 +26,15 @@ class TxConfirmStats; /* Identifier for each of the 3 different TxConfirmStats which will track * history over different time horizons. */ enum class FeeEstimateHorizon { - SHORT_HALFLIFE = 0, - MED_HALFLIFE = 1, - LONG_HALFLIFE = 2 + SHORT_HALFLIFE, + MED_HALFLIFE, + LONG_HALFLIFE, +}; + +static constexpr auto ALL_FEE_ESTIMATE_HORIZONS = std::array{ + FeeEstimateHorizon::SHORT_HALFLIFE, + FeeEstimateHorizon::MED_HALFLIFE, + FeeEstimateHorizon::LONG_HALFLIFE, }; std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon); diff --git a/src/protocol.cpp b/src/protocol.cpp index 56e738eaa8..0b893b9272 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,7 +5,6 @@ #include <protocol.h> -#include <util/strencodings.h> #include <util/system.h> static std::atomic<bool> g_initial_block_download_completed(false); @@ -86,7 +85,7 @@ const static std::string allNetMessageTypes[] = { NetMsgType::CFCHECKPT, NetMsgType::WTXIDRELAY, }; -const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes)); +const static std::vector<std::string> allNetMessageTypesVec(std::begin(allNetMessageTypes), std::end(allNetMessageTypes)); CMessageHeader::CMessageHeader() { diff --git a/src/psbt.cpp b/src/psbt.cpp index 4db57d3cd0..a849b2ea53 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -363,7 +363,7 @@ bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base6 bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) { - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss_data(MakeUCharSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/qt/README.md b/src/qt/README.md index 30c68db15b..20c712c98d 100644 --- a/src/qt/README.md +++ b/src/qt/README.md @@ -1,10 +1,12 @@ -This directory contains the BitcoinQT graphical user interface (GUI). It uses the cross-platform framework [Qt](https://www1.qt.io/developers/). +This directory contains the source code for the Bitcoin Core graphical user interface (GUI). It uses the [Qt](https://www1.qt.io/developers/) cross-platform framework. The current precise version for Qt 5 is specified in [qt.mk](/depends/packages/qt.mk). ## Compile and run -See build instructions ([macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [Unix](/doc/build-unix.md), etc). +See build instructions: [Unix](/doc/build-unix.md), [macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [FreeBSD](/doc/build-freebsd.md), [NetBSD](/doc/build-netbsd.md), [OpenBSD](/doc/build-openbsd.md) + +When following your systems build instructions, make sure to install the `Qt` dependencies. To run: @@ -12,84 +14,111 @@ To run: ./src/qt/bitcoin-qt ``` -## Files and directories - -### forms +## Files and Directories -Contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. They are created with [Qt Creator](#using-qt-creator-as-ide), but can be edited using any text editor. +#### forms/ -### locale +- A directory that contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. These files specify the characteristics of form elements in XML. Qt UI files can be edited with [Qt Creator](#using-qt-creator-as-ide) or using any text editor. -Contains translations. They are periodically updated. The process is described [here](/doc/translation_process.md). +#### locale/ -### res +- Contains translations. They are periodically updated and an effort is made to support as many languages as possible. The process of contributing translations is described in [doc/translation_process.md](/doc/translation_process.md). -Resources such as the icon. +#### res/ -### test + - Contains graphical resources used to enhance the UI experience. -Tests. +#### test/ -### bitcoingui.(h/cpp) +- Functional tests used to ensure proper functionality of the GUI. Significant changes to the GUI code normally require new or updated tests. -Represents the main window of the Bitcoin UI. +#### bitcoingui.(h/cpp) -### \*model.(h/cpp) +- Represents the main window of the Bitcoin UI. -The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +#### \*model.(h/cpp) -ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. +- The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +- ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. -### \*page.(h/cpp) +#### \*page.(h/cpp) -A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. +- A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. -### \*dialog.(h/cpp) +#### \*dialog.(h/cpp) -Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). +- Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). -### paymentserver.(h/cpp) +#### paymentserver.(h/cpp) -Used to process BIP21 payment URI requests. Also handles URI based application switching (e.g. when following a bitcoin:... link from a browser). +- (Deprecated) Used to process BIP21 payment URI requests. Also handles URI-based application switching (e.g. when following a bitcoin:... link from a browser). -### walletview.(h/cpp) +#### walletview.(h/cpp) -Represents the view to a single wallet. +- Represents the view to a single wallet. -### Other .h/cpp files +#### Other .h/cpp files * UI elements like BitcoinAmountField, which inherit from QWidget. * `bitcoinstrings.cpp`: automatically generated -* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc handling +* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc. handling * `callback.h` -* `guiconstants.h`: UI colors, app name, etc +* `guiconstants.h`: UI colors, app name, etc. * `guiutil.h`: several helper functions * `macdockiconhandler.(h/mm)`: macOS dock icon handler * `macnotificationhandler.(h/mm)`: display notifications in macOS ## Contribute -See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. Specifically for Qt: +See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. + +**Note:** Do not change `local/bitcoin_en.ts`. It is updated [automatically](/doc/translation_process.md#writing-code-with-translations). + +## Using Qt Creator as an IDE + +[Qt Creator](https://www.qt.io/product/development-tools) is a powerful tool which packages a UI designer tool (Qt Designer) and a C++ IDE into one application. This is especially useful if you want to change the UI layout. + +#### Download Qt Creator + +On Unix and macOS, Qt Creator can be installed through your package manager. Alternatively, you can download a binary from the [Qt Website](https://www.qt.io/download/). + +**Note:** If installing from a binary grabbed from the Qt Website: During the installation process, uncheck everything except for `Qt Creator`. + +##### macOS + +```sh +brew install qt-creator +``` + +##### Ubuntu & Debian + +```sh +sudo apt-get install qtcreator +``` + +#### Setup Qt Creator -* don't change `local/bitcoin_en.ts`; this happens [automatically](/doc/translation_process.md#writing-code-with-translations) +1. Make sure you've installed all dependencies specified in your systems build instructions +2. Follow the compile instructions for your system, run `./configure` with the `--enable-debug` flag +3. Start Qt Creator. At the start page, do: `New` -> `Import Project` -> `Import Existing Project` +4. Enter `bitcoin-qt` as the Project Name and enter the absolute path to `src/qt` as Location +5. Check over the file selection, you may need to select the `forms` directory (necessary if you intend to edit *.ui files) +6. Confirm the `Summary` page +7. In the `Projects` tab, select `Manage Kits...` -## Using Qt Creator as IDE + **macOS** + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"Clang (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"LLDB"` as debugger (you might need to set the path to your LLDB installation) -You can use Qt Creator as an IDE. This is especially useful if you want to change -the UI layout. + **Ubuntu & Debian** -Download and install the community edition of [Qt Creator](https://www.qt.io/download/). -Uncheck everything except Qt Creator during the installation process. + Note: Some of these options may already be set -Instructions for macOS: + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"GCC (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"GDB"` as debugger -1. Make sure you installed everything through Homebrew mentioned in the [macOS build instructions](/doc/build-osx.md) -2. Use `./configure` with the `--enable-debug` flag -3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project -4. Enter "bitcoin-qt" as project name, enter src/qt as location -5. Leave the file selection as it is -6. Confirm the "summary page" -7. In the "Projects" tab select "Manage Kits..." -8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler -9. Select LLDB as debugger (you might need to set the path to your installation) -10. Start debugging with Qt Creator (you might need to the executable to "bitcoin-qt" under "Run", which is where you can also add command line arguments) +8. While in the `Projects` tab, ensure that you have the `bitcoin-qt` executable specified under `Run` + - If the executable is not specified: click `"Choose..."`, navigate to `src/qt`, and select `bitcoin-qt` +9. You're all set! Start developing, building, and debugging the Bitcoin Core GUI diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 0fa52359a8..ab6168a541 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -56,7 +56,7 @@ protected: }; AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, Tabs _tab, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AddressBookPage), model(nullptr), mode(_mode), diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 8b1308da5d..bb444f22b3 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -11,7 +11,6 @@ #include <wallet/wallet.h> #include <algorithm> -#include <typeinfo> #include <QFont> #include <QDebug> @@ -82,8 +81,9 @@ public: { for (const auto& address : wallet.getAddresses()) { - if (pk_hash_only && address.dest.type() != typeid(PKHash)) + if (pk_hash_only && !std::holds_alternative<PKHash>(address.dest)) { continue; + } AddressTableEntry::Type addressType = translateTransactionType( QString::fromStdString(address.purpose), address.is_mine); cachedAddressTable.append(AddressTableEntry(addressType, @@ -261,7 +261,7 @@ bool AddressTableModel::setData(const QModelIndex &index, const QVariant &value, } else if(index.column() == Address) { CTxDestination newAddress = DecodeDestination(value.toString().toStdString()); // Refuse to set invalid address, set error status and return false - if(boost::get<CNoDestination>(&newAddress)) + if(std::get_if<CNoDestination>(&newAddress)) { editStatus = INVALID_ADDRESS; return false; diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index c2349b78f9..5b1330b81b 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -20,7 +20,7 @@ #include <QPushButton> AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureString* passphrase_out) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AskPassphraseDialog), mode(_mode), model(nullptr), diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8efb0e35d0..ef2f56c2c0 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -44,6 +44,7 @@ #include <QApplication> #include <QDebug> +#include <QFontDatabase> #include <QLibraryInfo> #include <QLocale> #include <QMessageBox> @@ -51,6 +52,7 @@ #include <QThread> #include <QTimer> #include <QTranslator> +#include <QtGlobal> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -462,11 +464,17 @@ int GuiMain(int argc, char* argv[]) // Generate high-dpi pixmaps QApplication::setAttribute(Qt::AA_UseHighDpiPixmaps); -#if QT_VERSION >= 0x050600 QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling); + +#if (QT_VERSION <= QT_VERSION_CHECK(5, 9, 8)) && defined(Q_OS_MACOS) + const auto os_name = QSysInfo::prettyProductName(); + if (os_name.startsWith("macOS 11") || os_name.startsWith("macOS 10.16")) { + QApplication::setStyle("fusion"); + } #endif BitcoinApplication app; + QFontDatabase::addApplicationFont(":/fonts/monospace"); /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: diff --git a/src/qt/bitcoin.qrc b/src/qt/bitcoin.qrc index 7115459808..fed373e551 100644 --- a/src/qt/bitcoin.qrc +++ b/src/qt/bitcoin.qrc @@ -83,4 +83,7 @@ <file alias="spinner-034">res/animation/spinner-034.png</file> <file alias="spinner-035">res/animation/spinner-035.png</file> </qresource> + <qresource prefix="/fonts"> + <file alias="monospace">res/fonts/RobotoMono-Bold.ttf</file> + </qresource> </RCC> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index f2a49e5a76..6677c9e3b5 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,6 +30,7 @@ #include <qt/macdockiconhandler.h> #endif +#include <functional> #include <chain.h> #include <chainparams.h> #include <interfaces/handler.h> @@ -87,6 +88,8 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty move(QGuiApplication::primaryScreen()->availableGeometry().center() - frameGeometry().center()); } + setContextMenuPolicy(Qt::PreventContextMenu); + #ifdef ENABLE_WALLET enableWallet = WalletModel::isWalletEnabled(); #endif // ENABLE_WALLET @@ -543,7 +546,6 @@ void BitcoinGUI::createToolBars() { QToolBar *toolbar = addToolBar(tr("Tabs toolbar")); appToolBar = toolbar; - toolbar->setContextMenuPolicy(Qt::PreventContextMenu); toolbar->setMovable(false); toolbar->setToolButtonStyle(Qt::ToolButtonTextBesideIcon); toolbar->addAction(overviewAction); @@ -845,7 +847,7 @@ void BitcoinGUI::showDebugWindowActivateConsole() void BitcoinGUI::showHelpMessageClicked() { - helpMessageDialog->show(); + GUIUtil::bringToFront(helpMessageDialog); } #ifdef ENABLE_WALLET diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a2f46c339b..56813b2d19 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include <validation.h> #include <stdint.h> +#include <functional> #include <QDebug> #include <QThread> diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 75fa970d37..ca78c96d70 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -42,7 +42,7 @@ bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { } CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _model, const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CoinControlDialog), m_coin_control(coin_control), model(_model), @@ -455,7 +455,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * else if(ExtractDestination(out.txout.scriptPubKey, address)) { CPubKey pubkey; - PKHash *pkhash = boost::get<PKHash>(&address); + PKHash* pkhash = std::get_if<PKHash>(&address); if (pkhash && model->wallet().getPubKey(out.txout.scriptPubKey, ToKeyID(*pkhash), pubkey)) { nBytesInputs += (pubkey.IsCompressed() ? 148 : 180); diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 70f080f4de..1467801522 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -9,10 +9,12 @@ #include <qt/createwalletdialog.h> #include <qt/forms/ui_createwalletdialog.h> +#include <qt/guiutil.h> + #include <QPushButton> CreateWalletDialog::CreateWalletDialog(QWidget* parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CreateWalletDialog) { ui->setupUi(this); diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index cd25f856a2..e51ed9e656 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -13,7 +13,7 @@ EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::EditAddressDialog), mapper(nullptr), mode(_mode), diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index 0b33c2cb8d..881869a46c 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -7,140 +7,135 @@ <x>0</x> <y>0</y> <width>364</width> - <height>213</height> + <height>249</height> </rect> </property> <property name="windowTitle"> <string>Create Wallet</string> </property> - <widget class="QDialogButtonBox" name="buttonBox"> - <property name="geometry"> - <rect> - <x>10</x> - <y>170</y> - <width>341</width> - <height>32</height> - </rect> - </property> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - <widget class="QLineEdit" name="wallet_name_line_edit"> - <property name="geometry"> - <rect> - <x>120</x> - <y>20</y> - <width>231</width> - <height>24</height> - </rect> - </property> - <property name="placeholderText"> - <string>Wallet</string> - </property> - </widget> - <widget class="QLabel" name="label"> - <property name="geometry"> - <rect> - <x>20</x> - <y>20</y> - <width>101</width> - <height>21</height> - </rect> - </property> - <property name="text"> - <string>Wallet Name</string> - </property> - </widget> - <widget class="QCheckBox" name="encrypt_wallet_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>50</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Encrypt the wallet. The wallet will be encrypted with a passphrase of your choice.</string> - </property> - <property name="text"> - <string>Encrypt Wallet</string> - </property> - <property name="checked"> - <bool>false</bool> - </property> - </widget> - <widget class="QLabel" name="advanced_options_label"> - <property name="geometry"> - <rect> - <x>20</x> - <y>90</y> - <width>220</width> - <height>21</height> - </rect> - </property> - <property name="styleSheet"> - <string notr="true">font-weight:bold;</string> - </property> - <property name="text"> - <string>Advanced options</string> - </property> - </widget> - <widget class="QCheckBox" name="disable_privkeys_checkbox"> - <property name="enabled"> - <bool>true</bool> - </property> - <property name="geometry"> - <rect> - <x>20</x> - <y>115</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Disable private keys for this wallet. Wallets with private keys disabled will have no private keys and cannot have an HD seed or imported private keys. This is ideal for watch-only wallets.</string> - </property> - <property name="text"> - <string>Disable Private Keys</string> - </property> - </widget> - <widget class="QCheckBox" name="blank_wallet_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>135</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Make a blank wallet. Blank wallets do not initially have private keys or scripts. Private keys and addresses can be imported, or an HD seed can be set, at a later time.</string> - </property> - <property name="text"> - <string>Make Blank Wallet</string> - </property> - </widget> - <widget class="QCheckBox" name="descriptor_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>155</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Use descriptors for scriptPubKey management</string> - </property> - <property name="text"> - <string>Descriptor Wallet</string> - </property> - </widget> + <property name="sizeGripEnabled"> + <bool>true</bool> + </property> + <layout class="QVBoxLayout" name="verticalLayout"> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QLabel" name="wallet_name_label"> + <property name="text"> + <string>Wallet Name</string> + </property> + </widget> + </item> + <item> + <widget class="QLineEdit" name="wallet_name_line_edit"> + <property name="minimumSize"> + <size> + <width>262</width> + <height>0</height> + </size> + </property> + <property name="placeholderText"> + <string>Wallet</string> + </property> + </widget> + </item> + </layout> + </item> + <item> + <widget class="QCheckBox" name="encrypt_wallet_checkbox"> + <property name="toolTip"> + <string>Encrypt the wallet. The wallet will be encrypted with a passphrase of your choice.</string> + </property> + <property name="text"> + <string>Encrypt Wallet</string> + </property> + <property name="checked"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <spacer name="verticalSpacer_1"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeType"> + <enum>QSizePolicy::Fixed</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>8</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QGroupBox" name="groupBox"> + <property name="title"> + <string>Advanced Options</string> + </property> + <layout class="QVBoxLayout" name="verticalLayout_groupbox"> + <item> + <widget class="QCheckBox" name="disable_privkeys_checkbox"> + <property name="enabled"> + <bool>true</bool> + </property> + <property name="toolTip"> + <string>Disable private keys for this wallet. Wallets with private keys disabled will have no private keys and cannot have an HD seed or imported private keys. This is ideal for watch-only wallets.</string> + </property> + <property name="text"> + <string>Disable Private Keys</string> + </property> + </widget> + </item> + <item> + <widget class="QCheckBox" name="blank_wallet_checkbox"> + <property name="toolTip"> + <string>Make a blank wallet. Blank wallets do not initially have private keys or scripts. Private keys and addresses can be imported, or an HD seed can be set, at a later time.</string> + </property> + <property name="text"> + <string>Make Blank Wallet</string> + </property> + </widget> + </item> + <item> + <widget class="QCheckBox" name="descriptor_checkbox"> + <property name="toolTip"> + <string>Use descriptors for scriptPubKey management</string> + </property> + <property name="text"> + <string>Descriptor Wallet</string> + </property> + </widget> + </item> + </layout> + </widget> + </item> + <item> + <spacer name="verticalSpacer_2"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>0</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QDialogButtonBox" name="buttonBox"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="standardButtons"> + <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> + </property> + </widget> + </item> + </layout> </widget> <tabstops> <tabstop>wallet_name_line_edit</tabstop> diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index d8112117cc..e45cafe48a 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -988,7 +988,7 @@ </item> </layout> </widget> - <widget class="QWidget" name="widget_2" native="true"> + <widget class="QWidget" name="peersTabRightPanel" native="true"> <property name="sizePolicy"> <sizepolicy hsizetype="Minimum" vsizetype="Preferred"> <horstretch>0</horstretch> @@ -1077,14 +1077,17 @@ </widget> </item> <item row="1" column="0"> - <widget class="QLabel" name="label_23"> + <widget class="QLabel" name="peerConnectionTypeLabel"> + <property name="toolTip"> + <string>The direction and type of peer connection: %1</string> + </property> <property name="text"> - <string>Direction</string> + <string>Direction/Type</string> </property> </widget> </item> <item row="1" column="1"> - <widget class="QLabel" name="peerDirection"> + <widget class="QLabel" name="peerConnectionType"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -1195,13 +1198,65 @@ </widget> </item> <item row="6" column="0"> + <widget class="QLabel" name="peerRelayTxesLabel"> + <property name="toolTip"> + <string>Whether the peer requested us to relay transactions.</string> + </property> + <property name="text"> + <string>Wants Tx Relay</string> + </property> + </widget> + </item> + <item row="6" column="1"> + <widget class="QLabel" name="peerRelayTxes"> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>N/A</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="7" column="0"> + <widget class="QLabel" name="peerHighBandwidthLabel"> + <property name="toolTip"> + <string>High bandwidth BIP152 compact block relay: %1</string> + </property> + <property name="text"> + <string>High Bandwidth</string> + </property> + </widget> + </item> + <item row="7" column="1"> + <widget class="QLabel" name="peerHighBandwidth"> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>N/A</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="8" column="0"> <widget class="QLabel" name="label_29"> <property name="text"> <string>Starting Block</string> </property> </widget> </item> - <item row="6" column="1"> + <item row="8" column="1"> <widget class="QLabel" name="peerHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1217,14 +1272,14 @@ </property> </widget> </item> - <item row="7" column="0"> + <item row="9" column="0"> <widget class="QLabel" name="label_27"> <property name="text"> <string>Synced Headers</string> </property> </widget> </item> - <item row="7" column="1"> + <item row="9" column="1"> <widget class="QLabel" name="peerSyncHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1240,14 +1295,14 @@ </property> </widget> </item> - <item row="8" column="0"> + <item row="10" column="0"> <widget class="QLabel" name="label_25"> <property name="text"> <string>Synced Blocks</string> </property> </widget> </item> - <item row="8" column="1"> + <item row="10" column="1"> <widget class="QLabel" name="peerCommonHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1263,14 +1318,14 @@ </property> </widget> </item> - <item row="9" column="0"> + <item row="11" column="0"> <widget class="QLabel" name="label_22"> <property name="text"> <string>Connection Time</string> </property> </widget> </item> - <item row="9" column="1"> + <item row="11" column="1"> <widget class="QLabel" name="peerConnTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1286,14 +1341,66 @@ </property> </widget> </item> - <item row="10" column="0"> + <item row="12" column="0"> + <widget class="QLabel" name="peerLastBlockLabel"> + <property name="toolTip"> + <string>Elapsed time since a novel block passing initial validity checks was received from this peer.</string> + </property> + <property name="text"> + <string>Last Block</string> + </property> + </widget> + </item> + <item row="12" column="1"> + <widget class="QLabel" name="peerLastBlock"> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>N/A</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="13" column="0"> + <widget class="QLabel" name="peerLastTxLabel"> + <property name="toolTip"> + <string>Elapsed time since a novel transaction accepted into our mempool was received from this peer.</string> + </property> + <property name="text"> + <string>Last Tx</string> + </property> + </widget> + </item> + <item row="13" column="1"> + <widget class="QLabel" name="peerLastTx"> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>N/A</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="14" column="0"> <widget class="QLabel" name="label_15"> <property name="text"> <string>Last Send</string> </property> </widget> </item> - <item row="10" column="1"> + <item row="14" column="1"> <widget class="QLabel" name="peerLastSend"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1309,14 +1416,14 @@ </property> </widget> </item> - <item row="11" column="0"> + <item row="15" column="0"> <widget class="QLabel" name="label_19"> <property name="text"> <string>Last Receive</string> </property> </widget> </item> - <item row="11" column="1"> + <item row="15" column="1"> <widget class="QLabel" name="peerLastRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1332,14 +1439,14 @@ </property> </widget> </item> - <item row="12" column="0"> + <item row="16" column="0"> <widget class="QLabel" name="label_18"> <property name="text"> <string>Sent</string> </property> </widget> </item> - <item row="12" column="1"> + <item row="16" column="1"> <widget class="QLabel" name="peerBytesSent"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1355,14 +1462,14 @@ </property> </widget> </item> - <item row="13" column="0"> + <item row="17" column="0"> <widget class="QLabel" name="label_20"> <property name="text"> <string>Received</string> </property> </widget> </item> - <item row="13" column="1"> + <item row="17" column="1"> <widget class="QLabel" name="peerBytesRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1378,14 +1485,14 @@ </property> </widget> </item> - <item row="14" column="0"> + <item row="18" column="0"> <widget class="QLabel" name="label_26"> <property name="text"> <string>Ping Time</string> </property> </widget> </item> - <item row="14" column="1"> + <item row="18" column="1"> <widget class="QLabel" name="peerPingTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1401,7 +1508,7 @@ </property> </widget> </item> - <item row="15" column="0"> + <item row="19" column="0"> <widget class="QLabel" name="peerPingWaitLabel"> <property name="toolTip"> <string>The duration of a currently outstanding ping.</string> @@ -1411,7 +1518,7 @@ </property> </widget> </item> - <item row="15" column="1"> + <item row="19" column="1"> <widget class="QLabel" name="peerPingWait"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1427,14 +1534,14 @@ </property> </widget> </item> - <item row="16" column="0"> + <item row="20" column="0"> <widget class="QLabel" name="peerMinPingLabel"> <property name="text"> <string>Min Ping</string> </property> </widget> </item> - <item row="16" column="1"> + <item row="20" column="1"> <widget class="QLabel" name="peerMinPing"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1450,14 +1557,14 @@ </property> </widget> </item> - <item row="17" column="0"> + <item row="21" column="0"> <widget class="QLabel" name="label_timeoffset"> <property name="text"> <string>Time Offset</string> </property> </widget> </item> - <item row="17" column="1"> + <item row="21" column="1"> <widget class="QLabel" name="timeoffset"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1473,7 +1580,7 @@ </property> </widget> </item> - <item row="18" column="0"> + <item row="22" column="0"> <widget class="QLabel" name="peerMappedASLabel"> <property name="toolTip"> <string>The mapped Autonomous System used for diversifying peer selection.</string> @@ -1483,7 +1590,7 @@ </property> </widget> </item> - <item row="18" column="1"> + <item row="22" column="1"> <widget class="QLabel" name="peerMappedAS"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1499,7 +1606,7 @@ </property> </widget> </item> - <item row="19" column="0"> + <item row="23" column="0"> <spacer name="verticalSpacer_3"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 8181cc47e2..6d279540e9 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -706,6 +706,106 @@ </layout> </item> <item> + <widget class="QGroupBox" name="font_groupBox"> + <property name="title"> + <string>Monospaced font in the Overview tab:</string> + </property> + <layout class="QVBoxLayout" name="font_verticalLayout"> + <item> + <layout class="QHBoxLayout" name="embeddedFont_horizontalLayout"> + <item> + <widget class="QRadioButton" name="embeddedFont_radioButton"> + <property name="text"> + <string>embedded "%1"</string> + </property> + </widget> + </item> + <item> + <spacer name="embeddedFont_horizontalSpacer"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QVBoxLayout" name="embeddedFont_verticalLayout"> + <item> + <widget class="QLabel" name="embeddedFont_label_1"> + <property name="text"> + <string>111.11111111 BTC</string> + </property> + </widget> + </item> + <item> + <widget class="QLabel" name="embeddedFont_label_9"> + <property name="text"> + <string>909.09090909 BTC</string> + </property> + </widget> + </item> + </layout> + </item> + </layout> + </item> + <item> + <widget class="Line" name="font_line"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + </widget> + </item> + <item> + <layout class="QHBoxLayout" name="systemFont_horizontalLayout"> + <item> + <widget class="QRadioButton" name="systemFont_radioButton"> + <property name="text"> + <string>closest matching "%1"</string> + </property> + </widget> + </item> + <item> + <spacer name="systemFont_horizontalSpacer"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QVBoxLayout" name="systemFont_verticalLayout"> + <item> + <widget class="QLabel" name="systemFont_label_1"> + <property name="text"> + <string>111.11111111 BTC</string> + </property> + </widget> + </item> + <item> + <widget class="QLabel" name="systemFont_label_9"> + <property name="text"> + <string>909.09090909 BTC</string> + </property> + </widget> + </item> + </layout> + </item> + </layout> + </item> + </layout> + </widget> + </item> + <item> <spacer name="verticalSpacer_Display"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 4d3f90c484..b82143e1ba 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -116,13 +116,6 @@ </property> <item row="2" column="2"> <widget class="QLabel" name="labelWatchPending"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -142,13 +135,6 @@ </item> <item row="2" column="1"> <widget class="QLabel" name="labelUnconfirmed"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -168,13 +154,6 @@ </item> <item row="3" column="2"> <widget class="QLabel" name="labelWatchImmature"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -227,13 +206,6 @@ </item> <item row="3" column="1"> <widget class="QLabel" name="labelImmature"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -273,13 +245,6 @@ </item> <item row="5" column="1"> <widget class="QLabel" name="labelTotal"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -299,13 +264,6 @@ </item> <item row="5" column="2"> <widget class="QLabel" name="labelWatchTotal"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -342,13 +300,6 @@ </item> <item row="1" column="1"> <widget class="QLabel" name="labelBalance"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -368,13 +319,6 @@ </item> <item row="1" column="2"> <widget class="QLabel" name="labelWatchAvailable"> - <property name="font"> - <font> - <family>Monospace</family> - <weight>75</weight> - <bold>true</bold> - </font> - </property> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -504,7 +448,7 @@ </layout> </item> <item> - <widget class="QListView" name="listTransactions"> + <widget class="TransactionOverviewWidget" name="listTransactions"> <property name="styleSheet"> <string notr="true">QListView { background: transparent; }</string> </property> @@ -517,9 +461,15 @@ <property name="horizontalScrollBarPolicy"> <enum>Qt::ScrollBarAlwaysOff</enum> </property> + <property name="sizeAdjustPolicy"> + <enum>QAbstractScrollArea::AdjustToContents</enum> + </property> <property name="selectionMode"> <enum>QAbstractItemView::NoSelection</enum> </property> + <property name="uniformItemSizes"> + <bool>true</bool> + </property> </widget> </item> </layout> @@ -544,6 +494,13 @@ </item> </layout> </widget> + <customwidgets> + <customwidget> + <class>TransactionOverviewWidget</class> + <extends>QListView</extends> + <header>qt/transactionoverviewwidget.h</header> + </customwidget> + </customwidgets> <resources/> <connections/> </ui> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 8205b0506c..c70bd9f418 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -40,12 +40,14 @@ #include <QFontDatabase> #include <QFontMetrics> #include <QGuiApplication> +#include <QJsonObject> #include <QKeyEvent> #include <QLineEdit> #include <QList> #include <QLocale> #include <QMenu> #include <QMouseEvent> +#include <QPluginLoader> #include <QProgressDialog> #include <QScreen> #include <QSettings> @@ -76,8 +78,11 @@ QString dateTimeStr(qint64 nTime) return dateTimeStr(QDateTime::fromTime_t((qint32)nTime)); } -QFont fixedPitchFont() +QFont fixedPitchFont(bool use_embedded_font) { + if (use_embedded_font) { + return {"Roboto Mono"}; + } return QFontDatabase::systemFont(QFontDatabase::FixedFont); } @@ -468,120 +473,6 @@ bool LabelOutOfFocusEventFilter::eventFilter(QObject* watched, QEvent* event) return QObject::eventFilter(watched, event); } -void TableViewLastColumnResizingFixer::connectViewHeadersSignals() -{ - connect(tableView->horizontalHeader(), &QHeaderView::sectionResized, this, &TableViewLastColumnResizingFixer::on_sectionResized); - connect(tableView->horizontalHeader(), &QHeaderView::geometriesChanged, this, &TableViewLastColumnResizingFixer::on_geometriesChanged); -} - -// We need to disconnect these while handling the resize events, otherwise we can enter infinite loops. -void TableViewLastColumnResizingFixer::disconnectViewHeadersSignals() -{ - disconnect(tableView->horizontalHeader(), &QHeaderView::sectionResized, this, &TableViewLastColumnResizingFixer::on_sectionResized); - disconnect(tableView->horizontalHeader(), &QHeaderView::geometriesChanged, this, &TableViewLastColumnResizingFixer::on_geometriesChanged); -} - -// Setup the resize mode, handles compatibility for Qt5 and below as the method signatures changed. -// Refactored here for readability. -void TableViewLastColumnResizingFixer::setViewHeaderResizeMode(int logicalIndex, QHeaderView::ResizeMode resizeMode) -{ - tableView->horizontalHeader()->setSectionResizeMode(logicalIndex, resizeMode); -} - -void TableViewLastColumnResizingFixer::resizeColumn(int nColumnIndex, int width) -{ - tableView->setColumnWidth(nColumnIndex, width); - tableView->horizontalHeader()->resizeSection(nColumnIndex, width); -} - -int TableViewLastColumnResizingFixer::getColumnsWidth() -{ - int nColumnsWidthSum = 0; - for (int i = 0; i < columnCount; i++) - { - nColumnsWidthSum += tableView->horizontalHeader()->sectionSize(i); - } - return nColumnsWidthSum; -} - -int TableViewLastColumnResizingFixer::getAvailableWidthForColumn(int column) -{ - int nResult = lastColumnMinimumWidth; - int nTableWidth = tableView->horizontalHeader()->width(); - - if (nTableWidth > 0) - { - int nOtherColsWidth = getColumnsWidth() - tableView->horizontalHeader()->sectionSize(column); - nResult = std::max(nResult, nTableWidth - nOtherColsWidth); - } - - return nResult; -} - -// Make sure we don't make the columns wider than the table's viewport width. -void TableViewLastColumnResizingFixer::adjustTableColumnsWidth() -{ - disconnectViewHeadersSignals(); - resizeColumn(lastColumnIndex, getAvailableWidthForColumn(lastColumnIndex)); - connectViewHeadersSignals(); - - int nTableWidth = tableView->horizontalHeader()->width(); - int nColsWidth = getColumnsWidth(); - if (nColsWidth > nTableWidth) - { - resizeColumn(secondToLastColumnIndex,getAvailableWidthForColumn(secondToLastColumnIndex)); - } -} - -// Make column use all the space available, useful during window resizing. -void TableViewLastColumnResizingFixer::stretchColumnWidth(int column) -{ - disconnectViewHeadersSignals(); - resizeColumn(column, getAvailableWidthForColumn(column)); - connectViewHeadersSignals(); -} - -// When a section is resized this is a slot-proxy for ajustAmountColumnWidth(). -void TableViewLastColumnResizingFixer::on_sectionResized(int logicalIndex, int oldSize, int newSize) -{ - adjustTableColumnsWidth(); - int remainingWidth = getAvailableWidthForColumn(logicalIndex); - if (newSize > remainingWidth) - { - resizeColumn(logicalIndex, remainingWidth); - } -} - -// When the table's geometry is ready, we manually perform the stretch of the "Message" column, -// as the "Stretch" resize mode does not allow for interactive resizing. -void TableViewLastColumnResizingFixer::on_geometriesChanged() -{ - if ((getColumnsWidth() - this->tableView->horizontalHeader()->width()) != 0) - { - disconnectViewHeadersSignals(); - resizeColumn(secondToLastColumnIndex, getAvailableWidthForColumn(secondToLastColumnIndex)); - connectViewHeadersSignals(); - } -} - -/** - * Initializes all internal variables and prepares the - * the resize modes of the last 2 columns of the table and - */ -TableViewLastColumnResizingFixer::TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth, QObject *parent) : - QObject(parent), - tableView(table), - lastColumnMinimumWidth(lastColMinimumWidth), - allColumnsMinimumWidth(allColsMinimumWidth) -{ - columnCount = tableView->horizontalHeader()->count(); - lastColumnIndex = columnCount - 1; - secondToLastColumnIndex = columnCount - 2; - tableView->horizontalHeader()->setMinimumSectionSize(allColumnsMinimumWidth); - setViewHeaderResizeMode(secondToLastColumnIndex, QHeaderView::Interactive); - setViewHeaderResizeMode(lastColumnIndex, QHeaderView::Interactive); -} - #ifdef WIN32 fs::path static StartupShortcutPath() { @@ -764,6 +655,23 @@ QString NetworkToQString(Network net) assert(false); } +QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction) +{ + QString prefix; + if (prepend_direction) { + prefix = (conn_type == ConnectionType::INBOUND) ? QObject::tr("Inbound") : QObject::tr("Outbound") + " "; + } + switch (conn_type) { + case ConnectionType::INBOUND: return prefix; + case ConnectionType::OUTBOUND_FULL_RELAY: return prefix + QObject::tr("Full Relay"); + case ConnectionType::BLOCK_RELAY: return prefix + QObject::tr("Block Relay"); + case ConnectionType::MANUAL: return prefix + QObject::tr("Manual"); + case ConnectionType::FEELER: return prefix + QObject::tr("Feeler"); + case ConnectionType::ADDR_FETCH: return prefix + QObject::tr("Address Fetch"); + } // no default case, so the compiler can warn about missing cases + assert(false); +} + QString formatDurationStr(int secs) { QStringList strList; @@ -923,6 +831,20 @@ void LogQtInfo() const std::string plugin_link{"dynamic"}; #endif LogPrintf("Qt %s (%s), plugin=%s (%s)\n", qVersion(), qt_link, QGuiApplication::platformName().toStdString(), plugin_link); + const auto static_plugins = QPluginLoader::staticPlugins(); + if (static_plugins.empty()) { + LogPrintf("No static plugins.\n"); + } else { + LogPrintf("Static plugins:\n"); + for (const QStaticPlugin& p : static_plugins) { + QJsonObject meta_data = p.metaData(); + const std::string plugin_class = meta_data.take(QString("className")).toString().toStdString(); + const int plugin_version = meta_data.take(QString("version")).toInt(); + LogPrintf(" %s, version %d\n", plugin_class, plugin_version); + } + } + + LogPrintf("Style: %s / %s\n", QApplication::style()->objectName().toStdString(), QApplication::style()->metaObject()->className()); LogPrintf("System: %s, %s\n", QSysInfo::prettyProductName().toStdString(), QSysInfo::buildAbi().toStdString()); for (const QScreen* s : QGuiApplication::screens()) { LogPrintf("Screen: %s %dx%d, pixel ratio=%.1f\n", s->name().toStdString(), s->size().width(), s->size().height(), s->devicePixelRatio()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 4bef13efb5..7984aa1141 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -7,17 +7,18 @@ #include <amount.h> #include <fs.h> +#include <net.h> #include <netaddress.h> #include <QEvent> #include <QHeaderView> #include <QItemDelegate> +#include <QLabel> #include <QMessageBox> #include <QObject> #include <QProgressBar> #include <QString> #include <QTableView> -#include <QLabel> class QValidatedLineEdit; class SendCoinsRecipient; @@ -44,12 +45,15 @@ QT_END_NAMESPACE */ namespace GUIUtil { + // Use this flags to prevent a "What's This" button in the title bar of the dialog on Windows. + constexpr auto dialog_flags = Qt::WindowTitleHint | Qt::WindowSystemMenuHint | Qt::WindowCloseButtonHint; + // Create human-readable string from date QString dateTimeStr(const QDateTime &datetime); QString dateTimeStr(qint64 nTime); // Return a monospace font - QFont fixedPitchFont(); + QFont fixedPitchFont(bool use_embedded_font = false); // Set up widget for address void setupAddressWidget(QValidatedLineEdit *widget, QWidget *parent); @@ -177,45 +181,6 @@ namespace GUIUtil bool eventFilter(QObject* watched, QEvent* event) override; }; - /** - * Makes a QTableView last column feel as if it was being resized from its left border. - * Also makes sure the column widths are never larger than the table's viewport. - * In Qt, all columns are resizable from the right, but it's not intuitive resizing the last column from the right. - * Usually our second to last columns behave as if stretched, and when on stretch mode, columns aren't resizable - * interactively or programmatically. - * - * This helper object takes care of this issue. - * - */ - class TableViewLastColumnResizingFixer: public QObject - { - Q_OBJECT - - public: - TableViewLastColumnResizingFixer(QTableView* table, int lastColMinimumWidth, int allColsMinimumWidth, QObject *parent); - void stretchColumnWidth(int column); - - private: - QTableView* tableView; - int lastColumnMinimumWidth; - int allColumnsMinimumWidth; - int lastColumnIndex; - int columnCount; - int secondToLastColumnIndex; - - void adjustTableColumnsWidth(); - int getAvailableWidthForColumn(int column); - int getColumnsWidth(); - void connectViewHeadersSignals(); - void disconnectViewHeadersSignals(); - void setViewHeaderResizeMode(int logicalIndex, QHeaderView::ResizeMode resizeMode); - void resizeColumn(int nColumnIndex, int width); - - private Q_SLOTS: - void on_sectionResized(int logicalIndex, int oldSize, int newSize); - void on_geometriesChanged(); - }; - bool GetStartOnSystemStartup(); bool SetStartOnSystemStartup(bool fAutoStart); @@ -228,6 +193,9 @@ namespace GUIUtil /** Convert enum Network to QString */ QString NetworkToQString(Network net); + /** Convert enum ConnectionType to QString */ + QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction); + /** Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 235722d091..aa6b2665fa 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -119,7 +119,7 @@ int GetPruneTargetGB() } // namespace Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::Intro), thread(nullptr), signalled(false), diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index b1081f6aee..ee70c1bc30 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -22,7 +22,6 @@ static const struct { {"signet", QAPP_APP_NAME_SIGNET, 35, 15}, {"regtest", QAPP_APP_NAME_REGTEST, 160, 30}, }; -static const unsigned network_styles_count = sizeof(network_styles)/sizeof(*network_styles); // titleAddText needs to be const char* for tr() NetworkStyle::NetworkStyle(const QString &_appName, const int iconColorHueShift, const int iconColorSaturationReduction, const char *_titleAddText): @@ -81,14 +80,12 @@ NetworkStyle::NetworkStyle(const QString &_appName, const int iconColorHueShift, const NetworkStyle* NetworkStyle::instantiate(const std::string& networkId) { std::string titleAddText = networkId == CBaseChainParams::MAIN ? "" : strprintf("[%s]", networkId); - for (unsigned x=0; x<network_styles_count; ++x) - { - if (networkId == network_styles[x].networkId) - { + for (const auto& network_style : network_styles) { + if (networkId == network_style.networkId) { return new NetworkStyle( - network_styles[x].appName, - network_styles[x].iconColorHueShift, - network_styles[x].iconColorSaturationReduction, + network_style.appName, + network_style.iconColorHueShift, + network_style.iconColorSaturationReduction, titleAddText.c_str()); } } diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 9a3d43c2a6..10bf82d532 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -11,7 +11,7 @@ #include <QUrl> OpenURIDialog::OpenURIDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OpenURIDialog) { ui->setupUi(this); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 416b9c83c9..e6b9488344 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -29,7 +29,7 @@ #include <QTimer> OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OptionsDialog), model(nullptr), mapper(nullptr) @@ -144,6 +144,20 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->minimizeToTray->setEnabled(false); } + QFont embedded_font{GUIUtil::fixedPitchFont(true)}; + ui->embeddedFont_radioButton->setText(ui->embeddedFont_radioButton->text().arg(QFontInfo(embedded_font).family())); + embedded_font.setWeight(QFont::Bold); + ui->embeddedFont_label_1->setFont(embedded_font); + ui->embeddedFont_label_9->setFont(embedded_font); + + QFont system_font{GUIUtil::fixedPitchFont(false)}; + ui->systemFont_radioButton->setText(ui->systemFont_radioButton->text().arg(QFontInfo(system_font).family())); + system_font.setWeight(QFont::Bold); + ui->systemFont_label_1->setFont(system_font); + ui->systemFont_label_9->setFont(system_font); + // Checking the embeddedFont_radioButton automatically unchecks the systemFont_radioButton. + ui->systemFont_radioButton->setChecked(true); + GUIUtil::handleCloseWindowShortcut(this); } @@ -246,6 +260,7 @@ void OptionsDialog::setMapper() mapper->addMapping(ui->lang, OptionsModel::Language); mapper->addMapping(ui->unit, OptionsModel::DisplayUnit); mapper->addMapping(ui->thirdPartyTxUrls, OptionsModel::ThirdPartyTxUrls); + mapper->addMapping(ui->embeddedFont_radioButton, OptionsModel::UseEmbeddedMonospacedFont); } void OptionsDialog::setOkButtonState(bool fState) diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 1e0391a35c..d51a5b06ff 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -163,6 +163,12 @@ void OptionsModel::Init(bool resetSettings) addOverriddenOption("-lang"); language = settings.value("language").toString(); + + if (!settings.contains("UseEmbeddedMonospacedFont")) { + settings.setValue("UseEmbeddedMonospacedFont", "true"); + } + m_use_embedded_monospaced_font = settings.value("UseEmbeddedMonospacedFont").toBool(); + Q_EMIT useEmbeddedMonospacedFontChanged(m_use_embedded_monospaced_font); } /** Helper function to copy contents from one QSettings to another. @@ -326,6 +332,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return strThirdPartyTxUrls; case Language: return settings.value("language"); + case UseEmbeddedMonospacedFont: + return m_use_embedded_monospaced_font; case CoinControlFeatures: return fCoinControlFeatures; case Prune: @@ -453,6 +461,11 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in setRestartRequired(true); } break; + case UseEmbeddedMonospacedFont: + m_use_embedded_monospaced_font = value.toBool(); + settings.setValue("UseEmbeddedMonospacedFont", m_use_embedded_monospaced_font); + Q_EMIT useEmbeddedMonospacedFontChanged(m_use_embedded_monospaced_font); + break; case CoinControlFeatures: fCoinControlFeatures = value.toBool(); settings.setValue("fCoinControlFeatures", fCoinControlFeatures); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index f7171951a1..4d012a9b8f 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -59,6 +59,7 @@ public: DisplayUnit, // BitcoinUnits::Unit ThirdPartyTxUrls, // QString Language, // QString + UseEmbeddedMonospacedFont, // bool CoinControlFeatures, // bool ThreadsScriptVerif, // int Prune, // bool @@ -84,6 +85,7 @@ public: bool getMinimizeOnClose() const { return fMinimizeOnClose; } int getDisplayUnit() const { return nDisplayUnit; } QString getThirdPartyTxUrls() const { return strThirdPartyTxUrls; } + bool getUseEmbeddedMonospacedFont() const { return m_use_embedded_monospaced_font; } bool getCoinControlFeatures() const { return fCoinControlFeatures; } const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } @@ -107,6 +109,7 @@ private: QString language; int nDisplayUnit; QString strThirdPartyTxUrls; + bool m_use_embedded_monospaced_font; bool fCoinControlFeatures; /* settings that were overridden by command-line */ QString strOverriddenByCommandLine; @@ -120,6 +123,7 @@ Q_SIGNALS: void displayUnitChanged(int unit); void coinControlFeaturesChanged(bool); void showTrayIconChanged(bool); + void useEmbeddedMonospacedFontChanged(bool); }; #endif // BITCOIN_QT_OPTIONSMODEL_H diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index cca4dce624..7f12b1d2b5 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -12,6 +12,7 @@ #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/transactionfilterproxy.h> +#include <qt/transactionoverviewwidget.h> #include <qt/transactiontablemodel.h> #include <qt/walletmodel.h> @@ -21,6 +22,9 @@ #include <QPainter> #include <QStatusTipEvent> +#include <algorithm> +#include <map> + #define DECORATION_SIZE 54 #define NUM_ITEMS 5 @@ -34,7 +38,7 @@ public: QAbstractItemDelegate(parent), unit(BitcoinUnits::BTC), platformStyle(_platformStyle) { - + connect(this, &TxViewDelegate::width_changed, this, &TxViewDelegate::sizeHintChanged); } inline void paint(QPainter *painter, const QStyleOptionViewItem &option, @@ -67,13 +71,15 @@ public: painter->setPen(foreground); QRect boundingRect; - painter->drawText(addressRect, Qt::AlignLeft|Qt::AlignVCenter, address, &boundingRect); + painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); + int address_rect_min_width = boundingRect.width(); if (index.data(TransactionTableModel::WatchonlyRole).toBool()) { QIcon iconWatchonly = qvariant_cast<QIcon>(index.data(TransactionTableModel::WatchonlyDecorationRole)); QRect watchonlyRect(boundingRect.right() + 5, mainRect.top()+ypad+halfheight, 16, halfheight); iconWatchonly.paint(painter, watchonlyRect); + address_rect_min_width += 5 + watchonlyRect.width(); } if(amount < 0) @@ -94,23 +100,42 @@ public: { amountText = QString("[") + amountText + QString("]"); } - painter->drawText(amountRect, Qt::AlignRight|Qt::AlignVCenter, amountText); + + QRect amount_bounding_rect; + painter->drawText(amountRect, Qt::AlignRight | Qt::AlignVCenter, amountText, &amount_bounding_rect); painter->setPen(option.palette.color(QPalette::Text)); - painter->drawText(amountRect, Qt::AlignLeft|Qt::AlignVCenter, GUIUtil::dateTimeStr(date)); + QRect date_bounding_rect; + painter->drawText(amountRect, Qt::AlignLeft | Qt::AlignVCenter, GUIUtil::dateTimeStr(date), &date_bounding_rect); + + const int minimum_width = std::max(address_rect_min_width, amount_bounding_rect.width() + date_bounding_rect.width()); + const auto search = m_minimum_width.find(index.row()); + if (search == m_minimum_width.end() || search->second != minimum_width) { + m_minimum_width[index.row()] = minimum_width; + Q_EMIT width_changed(index); + } painter->restore(); } inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override { - return QSize(DECORATION_SIZE, DECORATION_SIZE); + const auto search = m_minimum_width.find(index.row()); + const int minimum_text_width = search == m_minimum_width.end() ? 0 : search->second; + return {DECORATION_SIZE + 8 + minimum_text_width, DECORATION_SIZE}; } int unit; - const PlatformStyle *platformStyle; +Q_SIGNALS: + //! An intermediate signal for emitting from the `paint() const` member function. + void width_changed(const QModelIndex& index) const; + +private: + const PlatformStyle* platformStyle; + mutable std::map<int, int> m_minimum_width; }; + #include <qt/overviewpage.moc> OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) : @@ -126,7 +151,6 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) // use a SingleColorIcon for the "out of sync warning" icon QIcon icon = platformStyle->SingleColorIcon(":/icons/warning"); - icon.addPixmap(icon.pixmap(QSize(64,64), QIcon::Normal), QIcon::Disabled); // also set the disabled icon because we are using a disabled QPushButton to work around missing HiDPI support of QLabel (https://bugreports.qt.io/browse/QTBUG-42503) ui->labelTransactionsStatus->setIcon(icon); ui->labelWalletStatus->setIcon(icon); @@ -136,7 +160,7 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) ui->listTransactions->setMinimumHeight(NUM_ITEMS * (DECORATION_SIZE + 2)); ui->listTransactions->setAttribute(Qt::WA_MacShowFocusRect, false); - connect(ui->listTransactions, &QListView::clicked, this, &OverviewPage::handleTransactionClicked); + connect(ui->listTransactions, &TransactionOverviewWidget::clicked, this, &OverviewPage::handleTransactionClicked); // start with displaying the "out of sync" warnings showOutOfSyncWarning(true); @@ -233,6 +257,9 @@ void OverviewPage::setClientModel(ClientModel *model) // Show warning, for example if this is a prerelease version connect(model, &ClientModel::alertsChanged, this, &OverviewPage::updateAlerts); updateAlerts(model->getStatusBarWarnings()); + + connect(model->getOptionsModel(), &OptionsModel::useEmbeddedMonospacedFontChanged, this, &OverviewPage::setMonospacedFont); + setMonospacedFont(model->getOptionsModel()->getUseEmbeddedMonospacedFont()); } } @@ -297,3 +324,17 @@ void OverviewPage::showOutOfSyncWarning(bool fShow) ui->labelWalletStatus->setVisible(fShow); ui->labelTransactionsStatus->setVisible(fShow); } + +void OverviewPage::setMonospacedFont(bool use_embedded_font) +{ + QFont f = GUIUtil::fixedPitchFont(use_embedded_font); + f.setWeight(QFont::Bold); + ui->labelBalance->setFont(f); + ui->labelUnconfirmed->setFont(f); + ui->labelImmature->setFont(f); + ui->labelTotal->setFont(f); + ui->labelWatchAvailable->setFont(f); + ui->labelWatchPending->setFont(f); + ui->labelWatchImmature->setFont(f); + ui->labelWatchTotal->setFont(f); +} diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index 578ef601fb..5158c81678 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -61,6 +61,7 @@ private Q_SLOTS: void updateAlerts(const QString &warnings); void updateWatchOnlyLabels(bool showWatchOnly); void handleOutOfSyncWarningClicks(); + void setMonospacedFont(bool use_embedded_font); }; #endif // BITCOIN_QT_OVERVIEWPAGE_H diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index b02e4076e2..5f518a67cd 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -29,6 +29,8 @@ bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombine return pLeft->nodeid < pRight->nodeid; case PeerTableModel::Address: return pLeft->addrName.compare(pRight->addrName) < 0; + case PeerTableModel::ConnectionType: + return pLeft->m_conn_type < pRight->m_conn_type; case PeerTableModel::Network: return pLeft->m_network < pRight->m_network; case PeerTableModel::Ping: @@ -163,6 +165,8 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const case Address: // prepend to peer address down-arrow symbol for inbound connection and up-arrow for outbound connection return QString(rec->nodeStats.fInbound ? "↓ " : "↑ ") + QString::fromStdString(rec->nodeStats.addrName); + case ConnectionType: + return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /* prepend_direction */ false); case Network: return GUIUtil::NetworkToQString(rec->nodeStats.m_network); case Ping: @@ -176,6 +180,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const } } else if (role == Qt::TextAlignmentRole) { switch (index.column()) { + case ConnectionType: case Network: return QVariant(Qt::AlignCenter); case Ping: @@ -185,6 +190,11 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const default: return QVariant(); } + } else if (role == StatsRole) { + switch (index.column()) { + case NetNodeId: return QVariant::fromValue(rec); + default: return QVariant(); + } } return QVariant(); @@ -220,11 +230,6 @@ QModelIndex PeerTableModel::index(int row, int column, const QModelIndex &parent return QModelIndex(); } -const CNodeCombinedStats *PeerTableModel::getNodeStats(int idx) -{ - return priv->index(idx); -} - void PeerTableModel::refresh() { Q_EMIT layoutAboutToBeChanged(); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 61a0132e00..0823235ec0 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -28,6 +28,7 @@ struct CNodeCombinedStats { CNodeStateStats nodeStateStats; bool fNodeStateStatsAvailable; }; +Q_DECLARE_METATYPE(CNodeCombinedStats*) class NodeLessThan { @@ -52,19 +53,23 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); - const CNodeCombinedStats *getNodeStats(int idx); int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); void stopAutoRefresh(); enum ColumnIndex { NetNodeId = 0, - Address = 1, - Network = 2, - Ping = 3, - Sent = 4, - Received = 5, - Subversion = 6 + Address, + ConnectionType, + Network, + Ping, + Sent, + Received, + Subversion + }; + + enum { + StatsRole = Qt::UserRole, }; /** @name Methods overridden from QAbstractTableModel @@ -83,7 +88,7 @@ public Q_SLOTS: private: interfaces::Node& m_node; - const QStringList columns{tr("Peer Id"), tr("Address"), tr("Network"), tr("Ping"), tr("Sent"), tr("Received"), tr("User Agent")}; + const QStringList columns{tr("Peer Id"), tr("Address"), tr("Type"), tr("Network"), tr("Ping"), tr("Sent"), tr("Received"), tr("User Agent")}; std::unique_ptr<PeerTablePriv> priv; QTimer *timer; }; diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index c6b80fd340..aab8d8e4af 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -23,7 +23,6 @@ static const struct { /* Other: linux, unix, ... */ {"other", true, true, false} }; -static const unsigned platform_styles_count = sizeof(platform_styles)/sizeof(*platform_styles); namespace { /* Local functions for colorizing single-color images */ @@ -121,15 +120,13 @@ QIcon PlatformStyle::TextColorIcon(const QIcon& icon) const const PlatformStyle *PlatformStyle::instantiate(const QString &platformId) { - for (unsigned x=0; x<platform_styles_count; ++x) - { - if (platformId == platform_styles[x].platformId) - { + for (const auto& platform_style : platform_styles) { + if (platformId == platform_style.platformId) { return new PlatformStyle( - platform_styles[x].platformId, - platform_styles[x].imagesOnButtons, - platform_styles[x].colorizeIcons, - platform_styles[x].useExtraSpacing); + platform_style.platformId, + platform_style.imagesOnButtons, + platform_style.colorizeIcons, + platform_style.useExtraSpacing); } } return nullptr; diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 58167d4bb4..55ab6046cf 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -19,7 +19,7 @@ PSBTOperationsDialog::PSBTOperationsDialog( - QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent), + QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), m_ui(new Ui::PSBTOperationsDialog), m_wallet_model(wallet_model), m_client_model(client_model) diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 1a9cd78d1e..0da12c84fd 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -8,6 +8,7 @@ #include <qt/forms/ui_receivecoinsdialog.h> #include <qt/addresstablemodel.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/receiverequestdialog.h> @@ -18,12 +19,12 @@ #include <QCursor> #include <QMessageBox> #include <QScrollBar> +#include <QSettings> #include <QTextDocument> ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveCoinsDialog), - columnResizingFixer(nullptr), model(nullptr), platformStyle(_platformStyle) { @@ -43,13 +44,15 @@ ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWid // context menu actions QAction *copyURIAction = new QAction(tr("Copy URI"), this); - QAction *copyLabelAction = new QAction(tr("Copy label"), this); - QAction *copyMessageAction = new QAction(tr("Copy message"), this); - QAction *copyAmountAction = new QAction(tr("Copy amount"), this); + QAction* copyAddressAction = new QAction(tr("Copy address"), this); + copyLabelAction = new QAction(tr("Copy label"), this); + copyMessageAction = new QAction(tr("Copy message"), this); + copyAmountAction = new QAction(tr("Copy amount"), this); // context menu contextMenu = new QMenu(this); contextMenu->addAction(copyURIAction); + contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyMessageAction); contextMenu->addAction(copyAmountAction); @@ -57,11 +60,28 @@ ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWid // context menu signals connect(ui->recentRequestsView, &QWidget::customContextMenuRequested, this, &ReceiveCoinsDialog::showMenu); connect(copyURIAction, &QAction::triggered, this, &ReceiveCoinsDialog::copyURI); + connect(copyAddressAction, &QAction::triggered, this, &ReceiveCoinsDialog::copyAddress); connect(copyLabelAction, &QAction::triggered, this, &ReceiveCoinsDialog::copyLabel); connect(copyMessageAction, &QAction::triggered, this, &ReceiveCoinsDialog::copyMessage); connect(copyAmountAction, &QAction::triggered, this, &ReceiveCoinsDialog::copyAmount); connect(ui->clearButton, &QPushButton::clicked, this, &ReceiveCoinsDialog::clear); + + QTableView* tableView = ui->recentRequestsView; + tableView->verticalHeader()->hide(); + tableView->setAlternatingRowColors(true); + tableView->setSelectionBehavior(QAbstractItemView::SelectRows); + tableView->setSelectionMode(QAbstractItemView::ContiguousSelection); + + QSettings settings; + if (!tableView->horizontalHeader()->restoreState(settings.value("RecentRequestsViewHeaderState").toByteArray())) { + tableView->setColumnWidth(RecentRequestsTableModel::Date, DATE_COLUMN_WIDTH); + tableView->setColumnWidth(RecentRequestsTableModel::Label, LABEL_COLUMN_WIDTH); + tableView->setColumnWidth(RecentRequestsTableModel::Amount, AMOUNT_MINIMUM_COLUMN_WIDTH); + tableView->horizontalHeader()->setMinimumSectionSize(MINIMUM_COLUMN_WIDTH); + tableView->horizontalHeader()->setStretchLastSection(true); + } + tableView->horizontalHeader()->setSortIndicator(RecentRequestsTableModel::Date, Qt::DescendingOrder); } void ReceiveCoinsDialog::setModel(WalletModel *_model) @@ -75,22 +95,10 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) updateDisplayUnit(); QTableView* tableView = ui->recentRequestsView; - - tableView->verticalHeader()->hide(); - tableView->setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff); tableView->setModel(_model->getRecentRequestsTableModel()); - tableView->setAlternatingRowColors(true); - tableView->setSelectionBehavior(QAbstractItemView::SelectRows); - tableView->setSelectionMode(QAbstractItemView::ContiguousSelection); - tableView->setColumnWidth(RecentRequestsTableModel::Date, DATE_COLUMN_WIDTH); - tableView->setColumnWidth(RecentRequestsTableModel::Label, LABEL_COLUMN_WIDTH); - tableView->setColumnWidth(RecentRequestsTableModel::Amount, AMOUNT_MINIMUM_COLUMN_WIDTH); - connect(tableView->selectionModel(), &QItemSelectionModel::selectionChanged, this, &ReceiveCoinsDialog::recentRequestsView_selectionChanged); - // Last 2 columns are set by the columnResizingFixer, when the table geometry is ready. - columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(tableView, AMOUNT_MINIMUM_COLUMN_WIDTH, DATE_COLUMN_WIDTH, this); if (model->wallet().getDefaultAddressType() == OutputType::BECH32) { ui->useBech32->setCheckState(Qt::Checked); @@ -110,6 +118,8 @@ void ReceiveCoinsDialog::setModel(WalletModel *_model) ReceiveCoinsDialog::~ReceiveCoinsDialog() { + QSettings settings; + settings.setValue("RecentRequestsViewHeaderState", ui->recentRequestsView->horizontalHeader()->saveState()); delete ui; } @@ -234,14 +244,6 @@ void ReceiveCoinsDialog::on_removeRequestButton_clicked() model->getRecentRequestsTableModel()->removeRows(firstIndex.row(), selection.length(), firstIndex.parent()); } -// We override the virtual resizeEvent of the QWidget to adjust tables column -// sizes as the tables width is proportional to the dialogs width. -void ReceiveCoinsDialog::resizeEvent(QResizeEvent *event) -{ - QWidget::resizeEvent(event); - columnResizingFixer->stretchColumnWidth(RecentRequestsTableModel::Message); -} - QModelIndex ReceiveCoinsDialog::selectedRow() { if(!model || !model->getRecentRequestsTableModel() || !ui->recentRequestsView->selectionModel()) @@ -267,9 +269,18 @@ void ReceiveCoinsDialog::copyColumnToClipboard(int column) // context menu void ReceiveCoinsDialog::showMenu(const QPoint &point) { - if (!selectedRow().isValid()) { + const QModelIndex sel = selectedRow(); + if (!sel.isValid()) { return; } + + // disable context menu actions when appropriate + const RecentRequestsTableModel* const submodel = model->getRecentRequestsTableModel(); + const RecentRequestEntry& req = submodel->entry(sel.row()); + copyLabelAction->setDisabled(req.recipient.label.isEmpty()); + copyMessageAction->setDisabled(req.recipient.message.isEmpty()); + copyAmountAction->setDisabled(req.recipient.amount == 0); + contextMenu->exec(QCursor::pos()); } @@ -286,6 +297,19 @@ void ReceiveCoinsDialog::copyURI() GUIUtil::setClipboard(uri); } +// context menu action: copy address +void ReceiveCoinsDialog::copyAddress() +{ + const QModelIndex sel = selectedRow(); + if (!sel.isValid()) { + return; + } + + const RecentRequestsTableModel* const submodel = model->getRecentRequestsTableModel(); + const QString address = submodel->entry(sel.row()).recipient.address; + GUIUtil::setClipboard(address); +} + // context menu action: copy label void ReceiveCoinsDialog::copyLabel() { diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 9b89bd6a8b..fbbccc5a33 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -51,14 +51,15 @@ public Q_SLOTS: private: Ui::ReceiveCoinsDialog *ui; - GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer; WalletModel *model; QMenu *contextMenu; + QAction* copyLabelAction; + QAction* copyMessageAction; + QAction* copyAmountAction; const PlatformStyle *platformStyle; QModelIndex selectedRow(); void copyColumnToClipboard(int column); - virtual void resizeEvent(QResizeEvent *event) override; private Q_SLOTS: void on_receiveButton_clicked(); @@ -69,6 +70,7 @@ private Q_SLOTS: void updateDisplayUnit(); void showMenu(const QPoint &point); void copyURI(); + void copyAddress(); void copyLabel(); void copyMessage(); void copyAmount(); diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index bef3edc5fe..78ae5c07da 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -19,7 +19,7 @@ #endif ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveRequestDialog), model(nullptr) { diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 18b913774b..03531a1381 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -181,7 +181,7 @@ void RecentRequestsTableModel::addNewRequest(const SendCoinsRecipient &recipient // called from ctor when loading from wallet void RecentRequestsTableModel::addNewRequest(const std::string &recipient) { - std::vector<char> data(recipient.begin(), recipient.end()); + std::vector<uint8_t> data(recipient.begin(), recipient.end()); CDataStream ss(data, SER_DISK, CLIENT_VERSION); RecentRequestEntry entry; diff --git a/src/qt/res/fonts/RobotoMono-Bold.ttf b/src/qt/res/fonts/RobotoMono-Bold.ttf Binary files differnew file mode 100644 index 0000000000..900fce6848 --- /dev/null +++ b/src/qt/res/fonts/RobotoMono-Bold.ttf diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 61f47e3ad3..5acf8b1cf0 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -16,9 +16,10 @@ #include <chainparams.h> #include <interfaces/node.h> #include <netbase.h> -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> #include <util/threadnames.h> @@ -459,7 +460,24 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->splitter->restoreState(settings.value("PeersTabSplitterSizes").toByteArray()); - QChar nonbreaking_hyphen(8209); + constexpr QChar nonbreaking_hyphen(8209); + const std::vector<QString> CONNECTION_TYPE_DOC{ + tr("Inbound: initiated by peer"), + tr("Outbound Full Relay: default"), + tr("Outbound Block Relay: does not relay transactions or addresses"), + tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") + .arg("addnode") + .arg(QString(nonbreaking_hyphen) + "addnode") + .arg(QString(nonbreaking_hyphen) + "connect"), + tr("Outbound Feeler: short-lived, for testing addresses"), + tr("Outbound Address Fetch: short-lived, for soliciting addresses")}; + const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"}; + ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list)); + const QString hb_list{"<ul><li>\"" + + tr("To") + "\" – " + tr("we selected the peer for high bandwidth relay") + "</li><li>\"" + + tr("From") + "\" – " + tr("the peer selected us for high bandwidth relay") + "</li><li>\"" + + tr("No") + "\" – " + tr("no high bandwidth relay selected") + "</li></ul>"}; + ui->peerHighBandwidthLabel->setToolTip(ui->peerHighBandwidthLabel->toolTip().arg(hb_list)); ui->dataDir->setToolTip(ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg(QString(nonbreaking_hyphen) + "blocksdir")); ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); @@ -1021,11 +1039,9 @@ void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) void RPCConsole::peerLayoutAboutToChange() { - QModelIndexList selected = ui->peerWidget->selectionModel()->selectedIndexes(); cachedNodeids.clear(); - for(int i = 0; i < selected.size(); i++) - { - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected.at(i).row()); + for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { + const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); cachedNodeids.append(stats->nodeStats.nodeid); } } @@ -1084,15 +1100,13 @@ void RPCConsole::peerLayoutChanged() void RPCConsole::updateDetailWidget() { - QModelIndexList selected_rows; - auto selection_model = ui->peerWidget->selectionModel(); - if (selection_model) selected_rows = selection_model->selectedRows(); - if (!clientModel || !clientModel->getPeerTableModel() || selected_rows.size() != 1) { - ui->detailWidget->hide(); + const QList<QModelIndex> selected_peers = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); + if (!clientModel || !clientModel->getPeerTableModel() || selected_peers.size() != 1) { + ui->peersTabRightPanel->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); return; } - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected_rows.first().row()); + const auto stats = selected_peers.first().data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); // update the detail ui with latest node information QString peerAddrDetails(QString::fromStdString(stats->nodeStats.addrName) + " "); peerAddrDetails += tr("(peer id: %1)").arg(QString::number(stats->nodeStats.nodeid)); @@ -1100,18 +1114,26 @@ void RPCConsole::updateDetailWidget() peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); - ui->peerLastSend->setText(stats->nodeStats.nLastSend ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastSend) : tr("never")); - ui->peerLastRecv->setText(stats->nodeStats.nLastRecv ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastRecv) : tr("never")); + ui->peerRelayTxes->setText(stats->nodeStats.fRelayTxes ? "Yes" : "No"); + QString bip152_hb_settings; + if (stats->nodeStats.m_bip152_highbandwidth_to) bip152_hb_settings += "To"; + if (stats->nodeStats.m_bip152_highbandwidth_from) bip152_hb_settings += (bip152_hb_settings == "" ? "From" : "/From"); + if (bip152_hb_settings == "") bip152_hb_settings = "No"; + ui->peerHighBandwidth->setText(bip152_hb_settings); + const int64_t time_now{GetSystemTimeInSeconds()}; + ui->peerConnTime->setText(GUIUtil::formatDurationStr(time_now - stats->nodeStats.nTimeConnected)); + ui->peerLastBlock->setText(TimeDurationField(time_now, stats->nodeStats.nLastBlockTime)); + ui->peerLastTx->setText(TimeDurationField(time_now, stats->nodeStats.nLastTXTime)); + ui->peerLastSend->setText(TimeDurationField(time_now, stats->nodeStats.nLastSend)); + ui->peerLastRecv->setText(TimeDurationField(time_now, stats->nodeStats.nLastRecv)); ui->peerBytesSent->setText(GUIUtil::formatBytes(stats->nodeStats.nSendBytes)); ui->peerBytesRecv->setText(GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); - ui->peerConnTime->setText(GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nTimeConnected)); ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.m_ping_usec)); - ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStats.m_ping_wait_usec)); ui->peerMinPing->setText(GUIUtil::formatPingTime(stats->nodeStats.m_min_ping_usec)); ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /* prepend_direction */ true)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == PF_NONE) { ui->peerPermissions->setText(tr("N/A")); @@ -1140,9 +1162,10 @@ void RPCConsole::updateDetailWidget() ui->peerCommonHeight->setText(tr("Unknown")); ui->peerHeight->setText(QString::number(stats->nodeStateStats.m_starting_height)); + ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStateStats.m_ping_wait_usec)); } - ui->detailWidget->show(); + ui->peersTabRightPanel->show(); } void RPCConsole::resizeEvent(QResizeEvent *event) @@ -1205,19 +1228,9 @@ void RPCConsole::banSelectedNode(int bantime) if (!clientModel) return; - // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); - for(int i = 0; i < nodes.count(); i++) - { - // Get currently selected peer address - NodeId id = nodes.at(i).data().toLongLong(); - - // Get currently selected peer address - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); - if (detailNodeRow < 0) return; - + for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { // Find possible nodes, ban it and clear the selected node - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); + const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); if (stats) { m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 5f308dc36d..27d4c42eb4 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -168,6 +168,11 @@ private: /** Update UI with latest network info from model. */ void updateNetworkState(); + /** Helper for the output of a time duration field. Inputs are UNIX epoch times. */ + QString TimeDurationField(uint64_t time_now, uint64_t time_at_event) const { + return time_at_event ? GUIUtil::formatDurationStr(time_now - time_at_event) : tr("Never"); + } + private Q_SLOTS: void updateAlerts(const QString& warnings); }; diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index e765e643a3..611f3584c3 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -55,7 +55,7 @@ int getIndexForConfTarget(int target) { } SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SendCoinsDialog), clientModel(nullptr), model(nullptr), @@ -973,6 +973,9 @@ SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QStri setStandardButtons(QMessageBox::Yes | QMessageBox::Cancel); setDefaultButton(QMessageBox::Cancel); yesButton = button(QMessageBox::Yes); + if (confirmButtonText.isEmpty()) { + confirmButtonText = yesButton->text(); + } updateYesButton(); connect(&countDownTimer, &QTimer::timeout, this, &SendConfirmationDialog::countDown); } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 8e241d3901..4fc2f57cd6 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -115,7 +115,7 @@ class SendConfirmationDialog : public QMessageBox Q_OBJECT public: - SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "Send", QWidget* parent = nullptr); + SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "", QWidget* parent = nullptr); int exec() override; private Q_SLOTS: diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4835dd7954..2e7df60574 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -19,7 +19,7 @@ #include <QClipboard> SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SignVerifyMessageDialog), model(nullptr), platformStyle(_platformStyle) @@ -120,7 +120,7 @@ void SignVerifyMessageDialog::on_signMessageButton_SM_clicked() ui->statusLabel_SM->setText(tr("The entered address is invalid.") + QString(" ") + tr("Please check the address and try again.")); return; } - const PKHash* pkhash = boost::get<PKHash>(&destination); + const PKHash* pkhash = std::get_if<PKHash>(&destination); if (!pkhash) { ui->addressIn_SM->setValid(false); ui->statusLabel_SM->setStyleSheet("QLabel { color: red; }"); diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 5796a6ef56..2292c01d6a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -18,6 +18,8 @@ #include <util/system.h> #include <util/translation.h> +#include <functional> + #include <QApplication> #include <QCloseEvent> #include <QPainter> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 24d7735447..0d9928d363 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -33,7 +33,7 @@ static RPCHelpMan rpcNestedTest_rpc() } static const CRPCCommand vRPCCommands[] = { - {"test", "rpcNestedTest", &rpcNestedTest_rpc, {"arg1", "arg2", "arg3"}}, + {"test", &rpcNestedTest_rpc}, }; void RPCNestedTests::rpcNestedTests() diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index b547bf7645..6f31d8463f 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -11,7 +11,7 @@ #include <QModelIndex> TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::TransactionDescDialog) { ui->setupUi(this); diff --git a/src/qt/transactionoverviewwidget.h b/src/qt/transactionoverviewwidget.h new file mode 100644 index 0000000000..2bdead7bc4 --- /dev/null +++ b/src/qt/transactionoverviewwidget.h @@ -0,0 +1,41 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H +#define BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H + +#include <qt/transactiontablemodel.h> + +#include <QListView> +#include <QSize> +#include <QSizePolicy> + +QT_BEGIN_NAMESPACE +class QShowEvent; +class QWidget; +QT_END_NAMESPACE + +class TransactionOverviewWidget : public QListView +{ + Q_OBJECT + +public: + explicit TransactionOverviewWidget(QWidget* parent = nullptr) : QListView(parent) {} + + QSize sizeHint() const override + { + return {sizeHintForColumn(TransactionTableModel::ToAddress), QListView::sizeHint().height()}; + } + +protected: + void showEvent(QShowEvent* event) override + { + Q_UNUSED(event); + QSizePolicy sp = sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Minimum); + setSizePolicy(sp); + } +}; + +#endif // BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 06108f1d07..77fec93f0f 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -123,7 +123,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const interface continue; } - if (!boost::get<CNoDestination>(&wtx.txout_address[nOut])) + if (!std::get_if<CNoDestination>(&wtx.txout_address[nOut])) { // Sent to Bitcoin Address sub.type = TransactionRecord::SendToAddress; diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 05b250eea9..e6d483364c 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -19,6 +19,7 @@ #include <uint256.h> #include <algorithm> +#include <functional> #include <QColor> #include <QDateTime> diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 0cf6480b82..e1ec7b6ed0 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -31,6 +31,7 @@ #include <QMenu> #include <QPoint> #include <QScrollBar> +#include <QSettings> #include <QTableView> #include <QTimer> #include <QUrl> @@ -126,27 +127,40 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa vlayout->setContentsMargins(0,0,0,0); vlayout->setSpacing(0); - QTableView *view = new QTableView(this); + transactionView = new QTableView(this); + transactionView->setObjectName("transactionView"); vlayout->addLayout(hlayout); vlayout->addWidget(createDateRangeWidget()); - vlayout->addWidget(view); + vlayout->addWidget(transactionView); vlayout->setSpacing(0); - int width = view->verticalScrollBar()->sizeHint().width(); + int width = transactionView->verticalScrollBar()->sizeHint().width(); // Cover scroll bar width with spacing if (platformStyle->getUseExtraSpacing()) { hlayout->addSpacing(width+2); } else { hlayout->addSpacing(width); } - // Always show scroll bar - view->setVerticalScrollBarPolicy(Qt::ScrollBarAlwaysOn); - view->setTabKeyNavigation(false); - view->setContextMenuPolicy(Qt::CustomContextMenu); - - view->installEventFilter(this); - - transactionView = view; - transactionView->setObjectName("transactionView"); + transactionView->setVerticalScrollBarPolicy(Qt::ScrollBarAlwaysOn); + transactionView->setTabKeyNavigation(false); + transactionView->setContextMenuPolicy(Qt::CustomContextMenu); + transactionView->installEventFilter(this); + transactionView->setAlternatingRowColors(true); + transactionView->setSelectionBehavior(QAbstractItemView::SelectRows); + transactionView->setSelectionMode(QAbstractItemView::ExtendedSelection); + transactionView->setSortingEnabled(true); + transactionView->verticalHeader()->hide(); + + QSettings settings; + if (!transactionView->horizontalHeader()->restoreState(settings.value("TransactionViewHeaderState").toByteArray())) { + transactionView->setColumnWidth(TransactionTableModel::Status, STATUS_COLUMN_WIDTH); + transactionView->setColumnWidth(TransactionTableModel::Watchonly, WATCHONLY_COLUMN_WIDTH); + transactionView->setColumnWidth(TransactionTableModel::Date, DATE_COLUMN_WIDTH); + transactionView->setColumnWidth(TransactionTableModel::Type, TYPE_COLUMN_WIDTH); + transactionView->setColumnWidth(TransactionTableModel::Amount, AMOUNT_MINIMUM_COLUMN_WIDTH); + transactionView->horizontalHeader()->setMinimumSectionSize(MINIMUM_COLUMN_WIDTH); + transactionView->horizontalHeader()->setStretchLastSection(true); + } + transactionView->horizontalHeader()->setSortIndicator(TransactionTableModel::Date, Qt::DescendingOrder); // Actions abandonAction = new QAction(tr("Abandon transaction"), this); @@ -158,7 +172,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *copyTxIDAction = new QAction(tr("Copy transaction ID"), this); QAction *copyTxHexAction = new QAction(tr("Copy raw transaction"), this); QAction *copyTxPlainText = new QAction(tr("Copy full transaction details"), this); - QAction *editLabelAction = new QAction(tr("Edit label"), this); + QAction *editLabelAction = new QAction(tr("Edit address label"), this); QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); contextMenu = new QMenu(this); @@ -183,8 +197,8 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(search_widget, &QLineEdit::textChanged, prefix_typing_delay, static_cast<void (QTimer::*)()>(&QTimer::start)); connect(prefix_typing_delay, &QTimer::timeout, this, &TransactionView::changedSearch); - connect(view, &QTableView::doubleClicked, this, &TransactionView::doubleClicked); - connect(view, &QTableView::customContextMenuRequested, this, &TransactionView::contextualMenu); + connect(transactionView, &QTableView::doubleClicked, this, &TransactionView::doubleClicked); + connect(transactionView, &QTableView::customContextMenuRequested, this, &TransactionView::contextualMenu); connect(bumpFeeAction, &QAction::triggered, this, &TransactionView::bumpFee); connect(abandonAction, &QAction::triggered, this, &TransactionView::abandonTx); @@ -204,6 +218,12 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa }); } +TransactionView::~TransactionView() +{ + QSettings settings; + settings.setValue("TransactionViewHeaderState", transactionView->horizontalHeader()->saveState()); +} + void TransactionView::setModel(WalletModel *_model) { this->model = _model; @@ -214,25 +234,8 @@ void TransactionView::setModel(WalletModel *_model) transactionProxyModel->setDynamicSortFilter(true); transactionProxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); transactionProxyModel->setFilterCaseSensitivity(Qt::CaseInsensitive); - transactionProxyModel->setSortRole(Qt::EditRole); - - transactionView->setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff); transactionView->setModel(transactionProxyModel); - transactionView->setAlternatingRowColors(true); - transactionView->setSelectionBehavior(QAbstractItemView::SelectRows); - transactionView->setSelectionMode(QAbstractItemView::ExtendedSelection); - transactionView->horizontalHeader()->setSortIndicator(TransactionTableModel::Date, Qt::DescendingOrder); - transactionView->setSortingEnabled(true); - transactionView->verticalHeader()->hide(); - - transactionView->setColumnWidth(TransactionTableModel::Status, STATUS_COLUMN_WIDTH); - transactionView->setColumnWidth(TransactionTableModel::Watchonly, WATCHONLY_COLUMN_WIDTH); - transactionView->setColumnWidth(TransactionTableModel::Date, DATE_COLUMN_WIDTH); - transactionView->setColumnWidth(TransactionTableModel::Type, TYPE_COLUMN_WIDTH); - transactionView->setColumnWidth(TransactionTableModel::Amount, AMOUNT_MINIMUM_COLUMN_WIDTH); - - columnResizingFixer = new GUIUtil::TableViewLastColumnResizingFixer(transactionView, AMOUNT_MINIMUM_COLUMN_WIDTH, MINIMUM_COLUMN_WIDTH, this); if (_model->getOptionsModel()) { @@ -623,14 +626,6 @@ void TransactionView::focusTransaction(const uint256& txid) } } -// We override the virtual resizeEvent of the QWidget to adjust tables column -// sizes as the tables width is proportional to the dialogs width. -void TransactionView::resizeEvent(QResizeEvent* event) -{ - QWidget::resizeEvent(event); - columnResizingFixer->stretchColumnWidth(TransactionTableModel::ToAddress); -} - // Need to override default Ctrl+C action for amount as default behaviour is just to copy DisplayRole text bool TransactionView::eventFilter(QObject *obj, QEvent *event) { diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index b268823066..35ada4aa7a 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -35,6 +35,7 @@ class TransactionView : public QWidget public: explicit TransactionView(const PlatformStyle *platformStyle, QWidget *parent = nullptr); + ~TransactionView(); void setModel(WalletModel *model); @@ -82,10 +83,6 @@ private: QWidget *createDateRangeWidget(); - GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer{nullptr}; - - virtual void resizeEvent(QResizeEvent* event) override; - bool eventFilter(QObject *obj, QEvent *event) override; private Q_SLOTS: diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index b7f85446f4..05499b2a41 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -29,7 +29,7 @@ /** "Help message" or "About" dialog box */ HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::HelpMessageDialog) { ui->setupUi(this); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 2edb7eff8a..02b3c62867 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -106,9 +106,24 @@ void WalletFrame::setCurrentWallet(WalletModel* wallet_model) { if (mapWalletViews.count(wallet_model) == 0) return; + // Stop the effect of hidden widgets on the size hint of the shown one in QStackedWidget. + WalletView* view_about_to_hide = currentWalletView(); + if (view_about_to_hide) { + QSizePolicy sp = view_about_to_hide->sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Ignored); + view_about_to_hide->setSizePolicy(sp); + } + WalletView *walletView = mapWalletViews.value(wallet_model); - walletStack->setCurrentWidget(walletView); assert(walletView); + + // Set or restore the default QSizePolicy which could be set to QSizePolicy::Ignored previously. + QSizePolicy sp = walletView->sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Preferred); + walletView->setSizePolicy(sp); + walletView->updateGeometry(); + + walletStack->setCurrentWidget(walletView); walletView->updateEncryptionStatus(); } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index cad472b43b..02254da3ce 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -29,6 +29,7 @@ #include <wallet/wallet.h> // for CRecipient #include <stdint.h> +#include <functional> #include <QDebug> #include <QMessageBox> @@ -244,7 +245,7 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; - transaction_array.append(&(ssTx[0]), ssTx.size()); + transaction_array.append((const char*)&(ssTx[0]), ssTx.size()); } // Add addresses / update labels that we've sent to the address book, @@ -514,6 +515,13 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append("</td><td>"); questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee)); questionString.append("</td></tr></table>"); + + // Display warning in the "Confirm fee bump" window if the "Coin Control Features" option is enabled + if (getOptionsModel()->getCoinControlFeatures()) { + questionString.append("<br><br>"); + questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); + } + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); diff --git a/src/random.cpp b/src/random.cpp index af9504e0ce..9900825abb 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -38,7 +38,6 @@ #include <sys/random.h> #endif #ifdef HAVE_SYSCTL_ARND -#include <util/strencodings.h> // for ARRAYLEN #include <sys/sysctl.h> #endif @@ -333,7 +332,7 @@ void GetOSRand(unsigned char *ent32) int have = 0; do { size_t len = NUM_OS_RANDOM_BYTES - have; - if (sysctl(name, ARRAYLEN(name), ent32 + have, &len, nullptr, 0) != 0) { + if (sysctl(name, std::size(name), ent32 + have, &len, nullptr, 0) != 0) { RandFailure(); } have += len; diff --git a/src/rest.cpp b/src/rest.cpp index 678ffdb760..71426a4dc4 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -19,7 +19,6 @@ #include <txmempool.h> #include <util/check.h> #include <util/ref.h> -#include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -117,9 +116,10 @@ static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) param = strReq.substr(0, pos); const std::string suff(strReq, pos + 1); - for (unsigned int i = 0; i < ARRAYLEN(rf_names); i++) - if (suff == rf_names[i].name) - return rf_names[i].rf; + for (const auto& rf_name : rf_names) { + if (suff == rf_name.name) + return rf_name.rf; + } /* If no suffix is found, return original string. */ param = strReq; @@ -129,12 +129,13 @@ static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) static std::string AvailableDataFormatsString() { std::string formats; - for (unsigned int i = 0; i < ARRAYLEN(rf_names); i++) - if (strlen(rf_names[i].name) > 0) { + for (const auto& rf_name : rf_names) { + if (strlen(rf_name.name) > 0) { formats.append("."); - formats.append(rf_names[i].name); + formats.append(rf_name.name); formats.append(", "); } + } if (formats.length() > 0) return formats.substr(0, formats.length() - 2); @@ -179,7 +180,7 @@ static bool rest_headers(const util::Ref& context, { LOCK(cs_main); tip = ::ChainActive().Tip(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && ::ChainActive().Contains(pindex)) { headers.push_back(pindex); if (headers.size() == (unsigned long)count) @@ -247,7 +248,7 @@ static bool rest_block(HTTPRequest* req, { LOCK(cs_main); tip = ::ChainActive().Tip(); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -695,6 +696,7 @@ void InterruptREST() void StopREST() { - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) - UnregisterHTTPHandler(uri_prefixes[i].prefix, false); + for (const auto& up : uri_prefixes) { + UnregisterHTTPHandler(up.prefix, false); + } } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 689a8165ab..c71d2cbe49 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -17,8 +17,8 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> -#include <policy/fees.h> #include <policy/feerate.h> +#include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> #include <primitives/transaction.h> @@ -156,21 +156,11 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) { - // Serialize passed information without accessing chain state of the active chain! - AssertLockNotHeld(cs_main); // For performance reasons + UniValue result = blockheaderToJSON(tip, blockindex); - UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); - const CBlockIndex* pnext; - int confirmations = ComputeNextBlockAndDepth(tip, blockindex, pnext); - result.pushKV("confirmations", confirmations); result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", block.nVersion); - result.pushKV("versionHex", strprintf("%08x", block.nVersion)); - result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); UniValue txs(UniValue::VARR); if (txDetails) { CBlockUndo blockUndo; @@ -189,18 +179,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn } } result.pushKV("tx", txs); - result.pushKV("time", block.GetBlockTime()); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)block.nNonce); - result.pushKV("bits", strprintf("%08x", block.nBits)); - result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); - result.pushKV("nTx", (uint64_t)blockindex->nTx); - if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); - if (pnext) - result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } @@ -821,8 +800,8 @@ static RPCHelpMan getblockheader() {RPCResult::Type::NUM, "difficulty", "The difficulty"}, {RPCResult::Type::STR_HEX, "chainwork", "Expected number of hashes required to produce the current chain"}, {RPCResult::Type::NUM, "nTx", "The number of transactions in the block"}, - {RPCResult::Type::STR_HEX, "previousblockhash", "The hash of the previous block"}, - {RPCResult::Type::STR_HEX, "nextblockhash", "The hash of the next block"}, + {RPCResult::Type::STR_HEX, "previousblockhash", /* optional */ true, "The hash of the previous block (if available)"}, + {RPCResult::Type::STR_HEX, "nextblockhash", /* optional */ true, "The hash of the next block (if available)"}, }}, RPCResult{"for verbose=false", RPCResult::Type::STR_HEX, "", "A string that is serialized, hex-encoded data for block 'hash'"}, @@ -843,7 +822,7 @@ static RPCHelpMan getblockheader() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); } @@ -929,8 +908,8 @@ static RPCHelpMan getblock() {RPCResult::Type::NUM, "difficulty", "The difficulty"}, {RPCResult::Type::STR_HEX, "chainwork", "Expected number of hashes required to produce the chain up to this block (in hex)"}, {RPCResult::Type::NUM, "nTx", "The number of transactions in the block"}, - {RPCResult::Type::STR_HEX, "previousblockhash", "The hash of the previous block"}, - {RPCResult::Type::STR_HEX, "nextblockhash", "The hash of the next block"}, + {RPCResult::Type::STR_HEX, "previousblockhash", /* optional */ true, "The hash of the previous block (if available)"}, + {RPCResult::Type::STR_HEX, "nextblockhash", /* optional */ true, "The hash of the next block (if available)"}, }}, RPCResult{"for verbosity = 2", RPCResult::Type::OBJ, "", "", @@ -967,7 +946,7 @@ static RPCHelpMan getblock() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); if (!pblockindex) { @@ -1047,13 +1026,26 @@ static RPCHelpMan pruneblockchain() }; } +CoinStatsHashType ParseHashType(const std::string& hash_type_input) +{ + if (hash_type_input == "hash_serialized_2") { + return CoinStatsHashType::HASH_SERIALIZED; + } else if (hash_type_input == "muhash") { + return CoinStatsHashType::MUHASH; + } else if (hash_type_input == "none") { + return CoinStatsHashType::NONE; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s is not a valid hash_type", hash_type_input)); + } +} + static RPCHelpMan gettxoutsetinfo() { return RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n", { - {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'none'."}, + {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -1063,7 +1055,8 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, - {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, + {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, + {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount"}, }}, @@ -1078,7 +1071,7 @@ static RPCHelpMan gettxoutsetinfo() CCoinsStats stats; ::ChainstateActive().ForceFlushStateToDisk(); - const CoinStatsHashType hash_type = ParseHashType(request.params[0], CoinStatsHashType::HASH_SERIALIZED); + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); NodeContext& node = EnsureNodeContext(request.context); @@ -1091,6 +1084,9 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); } + if (hash_type == CoinStatsHashType::MUHASH) { + ret.pushKV("muhash", stats.hashSerialized.GetHex()); + } ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { @@ -1164,7 +1160,7 @@ static RPCHelpMan gettxout() } } - const CBlockIndex* pindex = LookupBlockIndex(coins_view->GetBestBlock()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(coins_view->GetBestBlock()); ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); if (coin.nHeight == MEMPOOL_HEIGHT) { ret.pushKV("confirmations", 0); @@ -1283,7 +1279,7 @@ RPCHelpMan getblockchaininfo() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "chain", "current network name (main, test, regtest)"}, + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, @@ -1500,6 +1496,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); + ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); @@ -1520,6 +1517,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritizetransaction"}, {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, @@ -1557,7 +1555,7 @@ static RPCHelpMan preciousblock() { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1595,7 +1593,7 @@ static RPCHelpMan invalidateblock() CBlockIndex* pblockindex; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1603,7 +1601,7 @@ static RPCHelpMan invalidateblock() InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); } if (!state.IsValid()) { @@ -1634,7 +1632,7 @@ static RPCHelpMan reconsiderblock() { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + CBlockIndex* pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1643,7 +1641,7 @@ static RPCHelpMan reconsiderblock() } BlockValidationState state; - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); @@ -1689,7 +1687,7 @@ static RPCHelpMan getchaintxstats() } else { uint256 hash(ParseHashV(request.params[1], "blockhash")); LOCK(cs_main); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1867,7 +1865,7 @@ static RPCHelpMan getblockstats() pindex = ::ChainActive()[height]; } else { const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2330,7 +2328,7 @@ static RPCHelpMan getblockfilter() bool block_was_connected; { LOCK(cs_main); - block_index = LookupBlockIndex(block_hash); + block_index = g_chainman.m_blockman.LookupBlockIndex(block_hash); if (!block_index) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2399,10 +2397,10 @@ static RPCHelpMan dumptxoutset() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + const fs::path path = fsbridge::AbsPathJoin(GetDataDir(), request.params[0].get_str()); // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + const fs::path temppath = fsbridge::AbsPathJoin(GetDataDir(), request.params[0].get_str() + ".incomplete"); if (fs::exists(path)) { throw JSONRPCError( @@ -2413,10 +2411,21 @@ static RPCHelpMan dumptxoutset() FILE* file{fsbridge::fopen(temppath, "wb")}; CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + NodeContext& node = EnsureNodeContext(request.context); + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile); + fs::rename(temppath, path); + + result.pushKV("path", path.string()); + return result; +}, + }; +} + +UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) +{ std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats; CBlockIndex* tip; - NodeContext& node = EnsureNodeContext(request.context); { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2433,14 +2442,14 @@ static RPCHelpMan dumptxoutset() // LOCK(::cs_main); - ::ChainstateActive().ForceFlushStateToDisk(); + chainstate.ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + if (!GetUTXOStats(&chainstate.CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } - pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - tip = LookupBlockIndex(stats.hashBlock); + pcursor = std::unique_ptr<CCoinsViewCursor>(chainstate.CoinsDB().Cursor()); + tip = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock); CHECK_NONFATAL(tip); } @@ -2464,57 +2473,54 @@ static RPCHelpMan dumptxoutset() } afile.fclose(); - fs::rename(temppath, path); UniValue result(UniValue::VOBJ); result.pushKV("coins_written", stats.coins_count); result.pushKV("base_hash", tip->GetBlockHash().ToString()); result.pushKV("base_height", tip->nHeight); - result.pushKV("path", path.string()); + return result; -}, - }; } void RegisterBlockchainRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "blockchain", "getblockchaininfo", &getblockchaininfo, {} }, - { "blockchain", "getchaintxstats", &getchaintxstats, {"nblocks", "blockhash"} }, - { "blockchain", "getblockstats", &getblockstats, {"hash_or_height", "stats"} }, - { "blockchain", "getbestblockhash", &getbestblockhash, {} }, - { "blockchain", "getblockcount", &getblockcount, {} }, - { "blockchain", "getblock", &getblock, {"blockhash","verbosity|verbose"} }, - { "blockchain", "getblockhash", &getblockhash, {"height"} }, - { "blockchain", "getblockheader", &getblockheader, {"blockhash","verbose"} }, - { "blockchain", "getchaintips", &getchaintips, {} }, - { "blockchain", "getdifficulty", &getdifficulty, {} }, - { "blockchain", "getmempoolancestors", &getmempoolancestors, {"txid","verbose"} }, - { "blockchain", "getmempooldescendants", &getmempooldescendants, {"txid","verbose"} }, - { "blockchain", "getmempoolentry", &getmempoolentry, {"txid"} }, - { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, - { "blockchain", "getrawmempool", &getrawmempool, {"verbose", "mempool_sequence"} }, - { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, - { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, - { "blockchain", "savemempool", &savemempool, {} }, - { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, - - { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, - { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, - { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "blockchain", &getblockchaininfo, }, + { "blockchain", &getchaintxstats, }, + { "blockchain", &getblockstats, }, + { "blockchain", &getbestblockhash, }, + { "blockchain", &getblockcount, }, + { "blockchain", &getblock, }, + { "blockchain", &getblockhash, }, + { "blockchain", &getblockheader, }, + { "blockchain", &getchaintips, }, + { "blockchain", &getdifficulty, }, + { "blockchain", &getmempoolancestors, }, + { "blockchain", &getmempooldescendants, }, + { "blockchain", &getmempoolentry, }, + { "blockchain", &getmempoolinfo, }, + { "blockchain", &getrawmempool, }, + { "blockchain", &gettxout, }, + { "blockchain", &gettxoutsetinfo, }, + { "blockchain", &pruneblockchain, }, + { "blockchain", &savemempool, }, + { "blockchain", &verifychain, }, + + { "blockchain", &preciousblock, }, + { "blockchain", &scantxoutset, }, + { "blockchain", &getblockfilter, }, /* Not shown in help */ - { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, - { "hidden", "reconsiderblock", &reconsiderblock, {"blockhash"} }, - { "hidden", "waitfornewblock", &waitfornewblock, {"timeout"} }, - { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, - { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, - { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, - { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, + { "hidden", &invalidateblock, }, + { "hidden", &reconsiderblock, }, + { "hidden", &waitfornewblock, }, + { "hidden", &waitforblock, }, + { "hidden", &waitforblockheight, }, + { "hidden", &syncwithvalidationinterfacequeue, }, + { "hidden", &dumptxoutset, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index e4ce80400e..d8cae4dd24 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -6,6 +6,7 @@ #define BITCOIN_RPC_BLOCKCHAIN_H #include <amount.h> +#include <streams.h> #include <sync.h> #include <stdint.h> @@ -16,6 +17,7 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CBlockPolicyEstimator; +class CChainState; class CTxMemPool; class ChainstateManager; class UniValue; @@ -57,4 +59,10 @@ CTxMemPool& EnsureMemPool(const util::Ref& context); ChainstateManager& EnsureChainman(const util::Ref& context); CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context); +/** + * Helper to create UTXO snapshots given a chainstate and a file handle. + * @return a UniValue map containing metadata about the snapshot. + */ +UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); + #endif diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 042005b9a6..2b593cd10b 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -183,6 +183,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 4, "avoid_reuse"}, { "createwallet", 5, "descriptors"}, { "createwallet", 6, "load_on_startup"}, + { "createwallet", 7, "external_signer"}, { "loadwallet", 1, "load_on_startup"}, { "unloadwallet", 1, "load_on_startup"}, { "getnodeaddresses", 0, "count"}, @@ -210,14 +211,9 @@ public: CRPCConvertTable::CRPCConvertTable() { - const unsigned int n_elem = - (sizeof(vRPCConvertParams) / sizeof(vRPCConvertParams[0])); - - for (unsigned int i = 0; i < n_elem; i++) { - members.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramIdx)); - membersByName.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramName)); + for (const auto& cp : vRPCConvertParams) { + members.emplace(cp.methodName, cp.paramIdx); + membersByName.emplace(cp.methodName, cp.paramName); } } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 965b278bfa..50987a735b 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -375,7 +375,7 @@ static RPCHelpMan generateblock() LOCK(cs_main); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), block, g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) { throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); } } @@ -409,7 +409,7 @@ static RPCHelpMan getmininginfo() {RPCResult::Type::NUM, "difficulty", "The current difficulty"}, {RPCResult::Type::NUM, "networkhashps", "The network hashes per second"}, {RPCResult::Type::NUM, "pooledtx", "The size of the mempool"}, - {RPCResult::Type::STR, "chain", "current network name (main, test, regtest)"}, + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, }}, RPCExamples{ @@ -618,7 +618,7 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); uint256 hash = block.GetHash(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; @@ -632,7 +632,7 @@ static RPCHelpMan getblocktemplate() if (block.hashPrevBlock != pindexPrev->GetBlockHash()) return "inconclusive-not-best-prevblk"; BlockValidationState state; - TestBlockValidity(state, Params(), block, pindexPrev, false, true); + TestBlockValidity(state, Params(), ::ChainstateActive(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } @@ -658,11 +658,15 @@ static RPCHelpMan getblocktemplate() if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + if (!Params().IsTestChain()) { + if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + } - if (::ChainstateActive().IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + if (::ChainstateActive().IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + } + } static unsigned int nTransactionsUpdatedLast; const CTxMemPool& mempool = EnsureMemPool(request.context); @@ -714,6 +718,13 @@ static RPCHelpMan getblocktemplate() // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners? } + const Consensus::Params& consensusParams = Params().GetConsensus(); + + // GBT must be called with 'signet' set in the rules for signet chains + if (consensusParams.signet_blocks && setClientRules.count("signet") != 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "getblocktemplate must be called with the signet rule set (call with {\"rules\": [\"segwit\", \"signet\"]})"); + } + // GBT must be called with 'segwit' set in the rules if (setClientRules.count("segwit") != 1) { throw JSONRPCError(RPC_INVALID_PARAMETER, "getblocktemplate must be called with the segwit rule set (call with {\"rules\": [\"segwit\"]})"); @@ -745,7 +756,6 @@ static RPCHelpMan getblocktemplate() } CHECK_NONFATAL(pindexPrev); CBlock* pblock = &pblocktemplate->block; // pointer for convenience - const Consensus::Params& consensusParams = Params().GetConsensus(); // Update nTime UpdateTime(pblock, consensusParams, pindexPrev); @@ -809,6 +819,12 @@ static RPCHelpMan getblocktemplate() UniValue aRules(UniValue::VARR); aRules.push_back("csv"); if (!fPreSegWit) aRules.push_back("!segwit"); + if (consensusParams.signet_blocks) { + // indicate to miner that they must understand signet rules + // when attempting to mine with this template + aRules.push_back("!signet"); + } + UniValue vbavailable(UniValue::VOBJ); for (int j = 0; j < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { Consensus::DeploymentPos pos = Consensus::DeploymentPos(j); @@ -889,6 +905,10 @@ static RPCHelpMan getblocktemplate() result.pushKV("bits", strprintf("%08x", pblock->nBits)); result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); + if (consensusParams.signet_blocks) { + result.pushKV("signet_challenge", HexStr(consensusParams.signet_challenge)); + } + if (!pblocktemplate->vchCoinbaseCommitment.empty()) { result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); } @@ -946,7 +966,7 @@ static RPCHelpMan submitblock() uint256 hash = block.GetHash(); { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { return "duplicate"; @@ -959,7 +979,7 @@ static RPCHelpMan submitblock() { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(block.hashPrevBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); if (pindex) { UpdateUncommittedBlockStructures(block, pindex, Params().GetConsensus()); } @@ -1003,7 +1023,7 @@ static RPCHelpMan submitheader() } { LOCK(cs_main); - if (!LookupBlockIndex(h.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(h.hashPrevBlock)) { throw JSONRPCError(RPC_VERIFY_ERROR, "Must submit previous header (" + h.hashPrevBlock.GetHex() + ") first"); } } @@ -1160,7 +1180,7 @@ static RPCHelpMan estimaterawfee() UniValue result(UniValue::VOBJ); - for (const FeeEstimateHorizon horizon : {FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}) { + for (const FeeEstimateHorizon horizon : ALL_FEE_ESTIMATE_HORIZONS) { CFeeRate feeRate; EstimationResult buckets; @@ -1212,24 +1232,24 @@ void RegisterMiningRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "mining", "getnetworkhashps", &getnetworkhashps, {"nblocks","height"} }, - { "mining", "getmininginfo", &getmininginfo, {} }, - { "mining", "prioritisetransaction", &prioritisetransaction, {"txid","dummy","fee_delta"} }, - { "mining", "getblocktemplate", &getblocktemplate, {"template_request"} }, - { "mining", "submitblock", &submitblock, {"hexdata","dummy"} }, - { "mining", "submitheader", &submitheader, {"hexdata"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "mining", &getnetworkhashps, }, + { "mining", &getmininginfo, }, + { "mining", &prioritisetransaction, }, + { "mining", &getblocktemplate, }, + { "mining", &submitblock, }, + { "mining", &submitheader, }, - { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, - { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, - { "generating", "generateblock", &generateblock, {"output","transactions"} }, + { "generating", &generatetoaddress, }, + { "generating", &generatetodescriptor, }, + { "generating", &generateblock, }, - { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, + { "util", &estimatesmartfee, }, - { "hidden", "estimaterawfee", &estimaterawfee, {"conf_target", "threshold"} }, - { "hidden", "generate", &generate, {} }, + { "hidden", &estimaterawfee, }, + { "hidden", &generate, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index b3102a236d..143be1274e 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -39,13 +39,14 @@ static RPCHelpMan validateaddress() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not. If not, this is the only property returned."}, + {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, {RPCResult::Type::STR, "address", "The bitcoin address validated"}, {RPCResult::Type::STR_HEX, "scriptPubKey", "The hex-encoded scriptPubKey generated by the address"}, {RPCResult::Type::BOOL, "isscript", "If the key is a script"}, {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address"}, {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, + {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, } }, RPCExamples{ @@ -54,13 +55,14 @@ static RPCHelpMan validateaddress() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - bool isValid = IsValidDestination(dest); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + const bool isValid = IsValidDestination(dest); + CHECK_NONFATAL(isValid == error_msg.empty()); UniValue ret(UniValue::VOBJ); ret.pushKV("isvalid", isValid); - if (isValid) - { + if (isValid) { std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -69,7 +71,10 @@ static RPCHelpMan validateaddress() UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); + } else { + ret.pushKV("error", error_msg); } + return ret; }, }; @@ -300,11 +305,11 @@ static RPCHelpMan verifymessage() switch (MessageVerify(strAddress, strSign, strMessage)) { case MessageVerificationResult::ERR_INVALID_ADDRESS: - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid address"); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); case MessageVerificationResult::ERR_ADDRESS_NO_KEY: throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); case MessageVerificationResult::ERR_MALFORMED_SIGNATURE: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Malformed base64 encoding"); + throw JSONRPCError(RPC_TYPE_ERROR, "Malformed base64 encoding"); case MessageVerificationResult::ERR_PUBKEY_NOT_RECOVERED: case MessageVerificationResult::ERR_NOT_SIGNED: return false; @@ -360,13 +365,13 @@ static RPCHelpMan signmessagewithprivkey() static RPCHelpMan setmocktime() { return RPCHelpMan{"setmocktime", - "\nSet the local time to given timestamp (-regtest only)\n", - { - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" - " Pass 0 to go back to using the system time."}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{""}, + "\nSet the local time to given timestamp (-regtest only)\n", + { + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" + "Pass 0 to go back to using the system time."}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { if (!Params().IsMockableChain()) { @@ -381,7 +386,10 @@ static RPCHelpMan setmocktime() LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - int64_t time = request.params[0].get_int64(); + const int64_t time{request.params[0].get_int64()}; + if (time < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime can not be negative: %s.", time)); + } SetMockTime(time); if (request.context.Has<NodeContext>()) { for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { @@ -694,23 +702,23 @@ void RegisterMiscRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "control", "getmemoryinfo", &getmemoryinfo, {"mode"} }, - { "control", "logging", &logging, {"include", "exclude"}}, - { "util", "validateaddress", &validateaddress, {"address"} }, - { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, - { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "range"} }, - { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, - { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, - { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, - { "util", "getindexinfo", &getindexinfo, {"index_name"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "control", &getmemoryinfo, }, + { "control", &logging, }, + { "util", &validateaddress, }, + { "util", &createmultisig, }, + { "util", &deriveaddresses, }, + { "util", &getdescriptorinfo, }, + { "util", &verifymessage, }, + { "util", &signmessagewithprivkey, }, + { "util", &getindexinfo, }, /* Not shown in help */ - { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, - { "hidden", "mockscheduler", &mockscheduler, {"delta_time"}}, - { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echojson, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", &setmocktime, }, + { "hidden", &mockscheduler, }, + { "hidden", &echo, }, + { "hidden", &echojson, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 89ddfb35cb..e80270d038 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -5,6 +5,7 @@ #include <rpc/server.h> #include <banman.h> +#include <chainparams.h> #include <clientversion.h> #include <core_io.h> #include <net.h> @@ -76,13 +77,12 @@ static RPCHelpMan ping() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { NodeContext& node = EnsureNodeContext(request.context); - if(!node.connman) + if (!node.peerman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } // Request that each node send a ping during next message processing pass - node.connman->ForEachNode([](CNode* pnode) { - pnode->fPingQueued = true; - }); + node.peerman->SendPings(); return NullUniValue; }, }; @@ -103,7 +103,7 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, {RPCResult::Type::STR, "addrbind", "(ip:port) Bind address of the connection to the peer"}, {RPCResult::Type::STR, "addrlocal", "(ip:port) Local address as reported by the peer"}, - {RPCResult::Type::STR, "network", "Network (ipv4, ipv6, or onion) the peer connected through"}, + {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, {RPCResult::Type::NUM, "mapped_as", "The AS in the BGP route to the peer used for diversifying\n" "peer selection (only available if the asmap config flag is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, @@ -128,9 +128,6 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, {RPCResult::Type::BOOL, "bip152_hb_to", "Whether we selected peer as (compact blocks) high-bandwidth peer"}, {RPCResult::Type::BOOL, "bip152_hb_from", "Whether peer selected us as (compact blocks) high-bandwidth peer"}, - {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" - "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" - "best capture connection behaviors."}, {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, @@ -156,6 +153,9 @@ static RPCHelpMan getpeerinfo() "Only known message types can appear as keys in the object and all bytes received\n" "of unknown message types are listed under '"+NET_MESSAGE_COMMAND_OTHER+"'."} }}, + {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" + "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" + "best capture connection behaviors."}, }}, }}, }, @@ -208,8 +208,8 @@ static RPCHelpMan getpeerinfo() if (stats.m_min_ping_usec < std::numeric_limits<int64_t>::max()) { obj.pushKV("minping", ((double)stats.m_min_ping_usec) / 1e6); } - if (stats.m_ping_wait_usec > 0) { - obj.pushKV("pingwait", ((double)stats.m_ping_wait_usec) / 1e6); + if (fStateStats && statestats.m_ping_wait_usec > 0) { + obj.pushKV("pingwait", ((double)statestats.m_ping_wait_usec) / 1e6); } obj.pushKV("version", stats.nVersion); // Use the sanitized form of subver here, to avoid tricksy remote peers from @@ -249,7 +249,7 @@ static RPCHelpMan getpeerinfo() recvPerMsgCmd.pushKV(i.first, i.second); } obj.pushKV("bytesrecv_per_msg", recvPerMsgCmd); - obj.pushKV("connection_type", stats.m_conn_type_string); + obj.pushKV("connection_type", ConnectionTypeAsString(stats.m_conn_type)); ret.push_back(obj); } @@ -314,6 +314,61 @@ static RPCHelpMan addnode() }; } +static RPCHelpMan addconnection() +{ + return RPCHelpMan{"addconnection", + "\nOpen an outbound connection to a specified node. This RPC is for testing only.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address and port to attempt connecting to."}, + {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open, either \"outbound-full-relay\" or \"block-relay-only\"."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + { RPCResult::Type::STR, "address", "Address of newly added connection." }, + { RPCResult::Type::STR, "connection_type", "Type of connection opened." }, + }}, + RPCExamples{ + HelpExampleCli("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"") + + HelpExampleRpc("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (Params().NetworkIDString() != CBaseChainParams::REGTEST) { + throw std::runtime_error("addconnection is for regression testing (-regtest mode) only."); + } + + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VSTR}); + const std::string address = request.params[0].get_str(); + const std::string conn_type_in{TrimString(request.params[1].get_str())}; + ConnectionType conn_type{}; + if (conn_type_in == "outbound-full-relay") { + conn_type = ConnectionType::OUTBOUND_FULL_RELAY; + } else if (conn_type_in == "block-relay-only") { + conn_type = ConnectionType::BLOCK_RELAY; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString()); + } + + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { + throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled."); + } + + const bool success = node.connman->AddConnection(address, conn_type); + if (!success) { + throw JSONRPCError(RPC_CLIENT_NODE_CAPACITY_REACHED, "Error: Already at capacity for specified connection type."); + } + + UniValue info(UniValue::VOBJ); + info.pushKV("address", address); + info.pushKV("connection_type", conn_type_in); + + return info; +}, + }; +} + static RPCHelpMan disconnectnode() { return RPCHelpMan{"disconnectnode", @@ -491,7 +546,7 @@ static UniValue GetNetworksInfo() UniValue networks(UniValue::VARR); for (int n = 0; n < NET_MAX; ++n) { enum Network network = static_cast<enum Network>(n); - if (network == NET_UNROUTABLE || network == NET_I2P || network == NET_CJDNS || network == NET_INTERNAL) continue; + if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); @@ -531,7 +586,7 @@ static RPCHelpMan getnetworkinfo() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "name", "network (ipv4, ipv6 or onion)"}, + {RPCResult::Type::STR, "name", "network (" + Join(GetNetworkNames(), ", ") + ")"}, {RPCResult::Type::BOOL, "limited", "is the network limited using -onlynet?"}, {RPCResult::Type::BOOL, "reachable", "is the network reachable?"}, {RPCResult::Type::STR, "proxy", "(\"host:port\") the proxy that is used for this network, or empty if none"}, @@ -885,22 +940,24 @@ void RegisterNetRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "network", "getconnectioncount", &getconnectioncount, {} }, - { "network", "ping", &ping, {} }, - { "network", "getpeerinfo", &getpeerinfo, {} }, - { "network", "addnode", &addnode, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, {"address", "nodeid"} }, - { "network", "getaddednodeinfo", &getaddednodeinfo, {"node"} }, - { "network", "getnettotals", &getnettotals, {} }, - { "network", "getnetworkinfo", &getnetworkinfo, {} }, - { "network", "setban", &setban, {"subnet", "command", "bantime", "absolute"} }, - { "network", "listbanned", &listbanned, {} }, - { "network", "clearbanned", &clearbanned, {} }, - { "network", "setnetworkactive", &setnetworkactive, {"state"} }, - { "network", "getnodeaddresses", &getnodeaddresses, {"count"} }, - { "hidden", "addpeeraddress", &addpeeraddress, {"address", "port"} }, +{ // category actor + // --------------------- ----------------------- + { "network", &getconnectioncount, }, + { "network", &ping, }, + { "network", &getpeerinfo, }, + { "network", &addnode, }, + { "network", &disconnectnode, }, + { "network", &getaddednodeinfo, }, + { "network", &getnettotals, }, + { "network", &getnetworkinfo, }, + { "network", &setban, }, + { "network", &listbanned, }, + { "network", &clearbanned, }, + { "network", &setnetworkactive, }, + { "network", &getnodeaddresses, }, + + { "hidden", &addconnection, }, + { "hidden", &addpeeraddress, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index d1475f452d..fc00a1efad 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -62,6 +62,7 @@ enum RPCErrorCode RPC_CLIENT_NODE_NOT_CONNECTED = -29, //!< Node to disconnect not found in connected nodes RPC_CLIENT_INVALID_IP_OR_SUBNET = -30, //!< Invalid IP/Subnet RPC_CLIENT_P2P_DISABLED = -31, //!< No valid connection manager instance found + RPC_CLIENT_NODE_CAPACITY_REACHED= -34, //!< Max number of outbound or block-relay connections already open //! Chain errors RPC_CLIENT_MEMPOOL_DISABLED = -33, //!< No mempool instance found @@ -78,6 +79,7 @@ enum RPCErrorCode RPC_WALLET_ALREADY_UNLOCKED = -17, //!< Wallet is already unlocked RPC_WALLET_NOT_FOUND = -18, //!< Invalid wallet specified RPC_WALLET_NOT_SPECIFIED = -19, //!< No wallet specified (error when there are multiple wallets loaded) + RPC_WALLET_ALREADY_LOADED = -35, //!< This same wallet is already loaded //! Backwards compatible aliases RPC_WALLET_INVALID_ACCOUNT_NAME = RPC_WALLET_INVALID_LABEL_NAME, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e2897549b5..47c776bbd1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -54,7 +54,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& LOCK(cs_main); entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = LookupBlockIndex(hashBlock); + CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (::ChainActive().Contains(pindex)) { entry.pushKV("confirmations", 1 + ::ChainActive().Height() - pindex->nHeight); @@ -178,7 +178,7 @@ static RPCHelpMan getrawtransaction() LOCK(cs_main); uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); - blockindex = LookupBlockIndex(blockhash); + blockindex = g_chainman.m_blockman.LookupBlockIndex(blockhash); if (!blockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } @@ -260,7 +260,7 @@ static RPCHelpMan gettxoutproof() if (!request.params[1].isNull()) { LOCK(cs_main); hashBlock = ParseHashV(request.params[1], "blockhash"); - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -290,7 +290,7 @@ static RPCHelpMan gettxoutproof() if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); } @@ -350,7 +350,7 @@ static RPCHelpMan verifytxoutproof() LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(merkleBlock.header.GetHash()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); if (!pindex || !::ChainActive().Contains(pindex) || pindex->nTx == 0) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -816,10 +816,11 @@ static RPCHelpMan sendrawtransaction() { return RPCHelpMan{"sendrawtransaction", "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" - "\nNote that the transaction will be sent unconditionally to all peers, so using this\n" + "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" - "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", + "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" + "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK()), @@ -894,6 +895,7 @@ static RPCHelpMan testmempoolaccept() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, {RPCResult::Type::BOOL, "allowed", "If the mempool allows this tx to be inserted"}, {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, {RPCResult::Type::OBJ, "fees", "Transaction fees (only present if 'allowed' is true)", @@ -930,7 +932,6 @@ static RPCHelpMan testmempoolaccept() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); } CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - const uint256& tx_hash = tx->GetHash(); const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? DEFAULT_MAX_RAW_TX_FEE_RATE : @@ -942,46 +943,38 @@ static RPCHelpMan testmempoolaccept() UniValue result(UniValue::VARR); UniValue result_0(UniValue::VOBJ); - result_0.pushKV("txid", tx_hash.GetHex()); + result_0.pushKV("txid", tx->GetHash().GetHex()); + result_0.pushKV("wtxid", tx->GetWitnessHash().GetHex()); - TxValidationState state; - bool test_accept_res; - CAmount fee{0}; - { - LOCK(cs_main); - test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), - nullptr /* plTxnReplaced */, false /* bypass_limits */, /* test_accept */ true, &fee); - } - - // Check that fee does not exceed maximum fee - if (test_accept_res && max_raw_tx_fee && fee > max_raw_tx_fee) { - result_0.pushKV("allowed", false); - result_0.pushKV("reject-reason", "max-fee-exceeded"); - result.push_back(std::move(result_0)); - return result; - } - result_0.pushKV("allowed", test_accept_res); + const MempoolAcceptResult accept_result = WITH_LOCK(cs_main, return AcceptToMemoryPool(::ChainstateActive(), mempool, std::move(tx), + false /* bypass_limits */, /* test_accept */ true)); // Only return the fee and vsize if the transaction would pass ATMP. // These can be used to calculate the feerate. - if (test_accept_res) { - result_0.pushKV("vsize", virtual_size); - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(fee)); - result_0.pushKV("fees", fees); + if (accept_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { + const CAmount fee = accept_result.m_base_fees.value(); + // Check that fee does not exceed maximum fee + if (max_raw_tx_fee && fee > max_raw_tx_fee) { + result_0.pushKV("allowed", false); + result_0.pushKV("reject-reason", "max-fee-exceeded"); + } else { + result_0.pushKV("allowed", true); + result_0.pushKV("vsize", virtual_size); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(fee)); + result_0.pushKV("fees", fees); + } + result.push_back(std::move(result_0)); } else { - if (state.IsInvalid()) { - if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { - result_0.pushKV("reject-reason", "missing-inputs"); - } else { - result_0.pushKV("reject-reason", strprintf("%s", state.GetRejectReason())); - } + result_0.pushKV("allowed", false); + const TxValidationState state = accept_result.m_state; + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_0.pushKV("reject-reason", "missing-inputs"); } else { result_0.pushKV("reject-reason", state.GetRejectReason()); } + result.push_back(std::move(result_0)); } - - result.push_back(std::move(result_0)); return result; }, }; @@ -1343,7 +1336,7 @@ static RPCHelpMan combinepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1482,7 +1475,7 @@ static RPCHelpMan createpsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1551,7 +1544,7 @@ static RPCHelpMan converttopsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1642,7 +1635,7 @@ static RPCHelpMan utxoupdatepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1738,7 +1731,7 @@ static RPCHelpMan joinpsbts() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << shuffled_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1858,27 +1851,27 @@ void RegisterRawTransactionRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose","blockhash"} }, - { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, - { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, - { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, - { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, - { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, - { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, - { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, - { "rawtransactions", "createpsbt", &createpsbt, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "converttopsbt", &converttopsbt, {"hexstring","permitsigdata","iswitness"} }, - { "rawtransactions", "utxoupdatepsbt", &utxoupdatepsbt, {"psbt", "descriptors"} }, - { "rawtransactions", "joinpsbts", &joinpsbts, {"txs"} }, - { "rawtransactions", "analyzepsbt", &analyzepsbt, {"psbt"} }, - - { "blockchain", "gettxoutproof", &gettxoutproof, {"txids", "blockhash"} }, - { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "rawtransactions", &getrawtransaction, }, + { "rawtransactions", &createrawtransaction, }, + { "rawtransactions", &decoderawtransaction, }, + { "rawtransactions", &decodescript, }, + { "rawtransactions", &sendrawtransaction, }, + { "rawtransactions", &combinerawtransaction, }, + { "rawtransactions", &signrawtransactionwithkey, }, + { "rawtransactions", &testmempoolaccept, }, + { "rawtransactions", &decodepsbt, }, + { "rawtransactions", &combinepsbt, }, + { "rawtransactions", &finalizepsbt, }, + { "rawtransactions", &createpsbt, }, + { "rawtransactions", &converttopsbt, }, + { "rawtransactions", &utxoupdatepsbt, }, + { "rawtransactions", &joinpsbts, }, + { "rawtransactions", &analyzepsbt, }, + + { "blockchain", &gettxoutproof, }, + { "blockchain", &verifytxoutproof, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index f32d9abac6..9a9b3713f3 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -144,8 +144,13 @@ static RPCHelpMan help() [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue { std::string strCommand; - if (jsonRequest.params.size() > 0) + if (jsonRequest.params.size() > 0) { strCommand = jsonRequest.params[0].get_str(); + } + if (strCommand == "dump_all_command_conversions") { + // Used for testing only, undocumented + return tableRPC.dumpArgMap(); + } return tableRPC.help(strCommand, jsonRequest); }, @@ -244,13 +249,13 @@ static RPCHelpMan getrpcinfo() // clang-format off static const CRPCCommand vRPCCommands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- +{ // category actor (function) + // --------------------- ----------------------- /* Overall control/query calls */ - { "control", "getrpcinfo", &getrpcinfo, {} }, - { "control", "help", &help, {"command"} }, - { "control", "stop", &stop, {"wait"} }, - { "control", "uptime", &uptime, {} }, + { "control", &getrpcinfo, }, + { "control", &help, }, + { "control", &stop, }, + { "control", &uptime, }, }; // clang-format on @@ -479,6 +484,18 @@ std::vector<std::string> CRPCTable::listCommands() const return commandList; } +UniValue CRPCTable::dumpArgMap() const +{ + UniValue ret{UniValue::VARR}; + for (const auto& cmd : mapCommands) { + for (const auto& c : cmd.second) { + const auto help = RpcMethodFnType(c->unique_id)(); + help.AppendArgMap(ret); + } + } + return ret; +} + void RPCSetTimerInterfaceIfUnset(RPCTimerInterface *iface) { if (!timerInterface) diff --git a/src/rpc/server.h b/src/rpc/server.h index 040192b455..fe5a791e1e 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -103,7 +103,7 @@ public: } //! Simplified constructor taking plain RpcMethodFnType function pointer. - CRPCCommand(std::string category, std::string name_in, RpcMethodFnType fn, std::vector<std::string> args_in) + CRPCCommand(std::string category, RpcMethodFnType fn) : CRPCCommand( category, fn().m_name, @@ -111,8 +111,6 @@ public: fn().GetArgNames(), intptr_t(fn)) { - CHECK_NONFATAL(fn().m_name == name_in); - CHECK_NONFATAL(fn().GetArgNames() == args_in); } std::string category; @@ -147,6 +145,10 @@ public: */ std::vector<std::string> listCommands() const; + /** + * Return all named arguments that need to be converted by the client from string to another JSON type + */ + UniValue dumpArgMap() const; /** * Appends a CRPCCommand to the dispatch table. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index e377a80fbd..e890c0108a 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,23 +113,6 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } -CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type) -{ - if (param.isNull()) { - return default_type; - } else { - std::string hash_type_input = param.get_str(); - - if (hash_type_input == "hash_serialized_2") { - return CoinStatsHashType::HASH_SERIALIZED; - } else if (hash_type_input == "none") { - return CoinStatsHashType::NONE; - } else { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%d is not a valid hash_type", hash_type_input)); - } - } -} - std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; @@ -209,7 +192,7 @@ CTxDestination AddAndGetMultisigDestination(const int required, const std::vecto return dest; } -class DescribeAddressVisitor : public boost::static_visitor<UniValue> +class DescribeAddressVisitor { public: explicit DescribeAddressVisitor() {} @@ -267,7 +250,7 @@ public: UniValue DescribeAddress(const CTxDestination& dest) { - return boost::apply_visitor(DescribeAddressVisitor(), dest); + return std::visit(DescribeAddressVisitor(), dest); } unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) @@ -549,6 +532,24 @@ std::string RPCHelpMan::ToString() const return ret; } +void RPCHelpMan::AppendArgMap(UniValue& arr) const +{ + for (int i{0}; i < int(m_args.size()); ++i) { + const auto& arg = m_args.at(i); + std::vector<std::string> arg_names; + boost::split(arg_names, arg.m_names, boost::is_any_of("|")); + for (const auto& arg_name : arg_names) { + UniValue map{UniValue::VARR}; + map.push_back(m_name); + map.push_back(i); + map.push_back(arg_name); + map.push_back(arg.m_type == RPCArg::Type::STR || + arg.m_type == RPCArg::Type::STR_HEX); + arr.push_back(map); + } + } +} + std::string RPCArg::GetFirstName() const { return m_names.substr(0, m_names.find("|")); diff --git a/src/rpc/util.h b/src/rpc/util.h index 942c243718..c54ce85f60 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -77,8 +77,6 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); -CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type); - extern CAmount AmountFromValue(const UniValue& value); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); @@ -336,6 +334,8 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); std::string ToString() const; + /** Append the named args that need to be converted from string to another JSON type */ + void AppendArgMap(UniValue& arr) const; UniValue HandleRequest(const JSONRPCRequest& request) { Check(request); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 7c361bf26f..b3ee23f139 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -7,6 +7,7 @@ #include <random.h> #include <assert.h> +#include <functional> #include <utility> CScheduler::CScheduler() diff --git a/src/scheduler.h b/src/scheduler.h index d7fe00d1b4..9eec8c0fa0 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -9,6 +9,7 @@ #include <functional> #include <list> #include <map> +#include <thread> #include <sync.h> @@ -35,6 +36,8 @@ public: CScheduler(); ~CScheduler(); + std::thread m_service_thread; + typedef std::function<void()> Function; /** Call func at/after time t */ @@ -62,8 +65,7 @@ public: void MockForward(std::chrono::seconds delta_seconds); /** - * Services the queue 'forever'. Should be run in a thread, - * and interrupted using boost::interrupt_thread + * Services the queue 'forever'. Should be run in a thread. */ void serviceQueue(); @@ -72,12 +74,14 @@ public: { WITH_LOCK(newTaskMutex, stopRequested = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** Tell any threads running serviceQueue to stop when there is no work left to be done */ void StopWhenDrained() { WITH_LOCK(newTaskMutex, stopWhenEmpty = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index c5dceac848..b6939127e1 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -11,14 +11,12 @@ #if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #if defined(_WIN32) - #if defined(DLL_EXPORT) - #if defined(HAVE_FUNC_ATTRIBUTE_DLLEXPORT) - #define EXPORT_SYMBOL __declspec(dllexport) - #else - #define EXPORT_SYMBOL - #endif + #if defined(HAVE_DLLEXPORT_ATTRIBUTE) + #define EXPORT_SYMBOL __declspec(dllexport) + #else + #define EXPORT_SYMBOL #endif - #elif defined(HAVE_FUNC_ATTRIBUTE_VISIBILITY) + #elif defined(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE) #define EXPORT_SYMBOL __attribute__ ((visibility ("default"))) #endif #elif defined(MSC_VER) && !defined(STATIC_LIBBITCOINCONSENSUS) diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index e5ba9ba6d2..6ab01882ac 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -179,6 +179,9 @@ public: /** Get the descriptor string form including private data (if available in arg). */ virtual bool ToPrivateString(const SigningProvider& arg, std::string& out) const = 0; + /** Get the descriptor string form with the xpub at the last hardened derivation */ + virtual bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const = 0; + /** Derive a private key, if private data is available in arg. */ virtual bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const = 0; }; @@ -212,6 +215,21 @@ public: ret = "[" + OriginString() + "]" + std::move(sub); return true; } + bool ToNormalizedString(const SigningProvider& arg, std::string& ret, bool priv) const override + { + std::string sub; + if (!m_provider->ToNormalizedString(arg, sub, priv)) return false; + // If m_provider is a BIP32PubkeyProvider, we may get a string formatted like a OriginPubkeyProvider + // In that case, we need to strip out the leading square bracket and fingerprint from the substring, + // and append that to our own origin string. + if (sub[0] == '[') { + sub = sub.substr(9); + ret = "[" + OriginString() + std::move(sub); + } else { + ret = "[" + OriginString() + "]" + std::move(sub); + } + return true; + } bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const override { return m_provider->GetPrivKey(pos, arg, key); @@ -243,6 +261,12 @@ public: ret = EncodeSecret(key); return true; } + bool ToNormalizedString(const SigningProvider& arg, std::string& ret, bool priv) const override + { + if (priv) return ToPrivateString(arg, ret); + ret = ToString(); + return true; + } bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const override { return arg.GetKey(m_pubkey.GetID(), key); @@ -386,6 +410,56 @@ public: } return true; } + bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const override + { + // For hardened derivation type, just return the typical string, nothing to normalize + if (m_derive == DeriveType::HARDENED) { + if (priv) return ToPrivateString(arg, out); + out = ToString(); + return true; + } + // Step backwards to find the last hardened step in the path + int i = (int)m_path.size() - 1; + for (; i >= 0; --i) { + if (m_path.at(i) >> 31) { + break; + } + } + // Either no derivation or all unhardened derivation + if (i == -1) { + if (priv) return ToPrivateString(arg, out); + out = ToString(); + return true; + } + // Derive the xpub at the last hardened step + CExtKey xprv; + if (!GetExtKey(arg, xprv)) return false; + KeyOriginInfo origin; + int k = 0; + for (; k <= i; ++k) { + // Derive + xprv.Derive(xprv, m_path.at(k)); + // Add to the path + origin.path.push_back(m_path.at(k)); + // First derivation element, get the fingerprint for origin + if (k == 0) { + std::copy(xprv.vchFingerprint, xprv.vchFingerprint + 4, origin.fingerprint); + } + } + // Build the remaining path + KeyPath end_path; + for (; k < (int)m_path.size(); ++k) { + end_path.push_back(m_path.at(k)); + } + // Build the string + std::string origin_str = HexStr(origin.fingerprint) + FormatHDKeypath(origin.path); + out = "[" + origin_str + "]" + (priv ? EncodeExtKey(xprv) : EncodeExtPubKey(xprv.Neuter())) + FormatHDKeypath(end_path); + if (IsRange()) { + out += "/*"; + assert(m_derive == DeriveType::UNHARDENED); + } + return true; + } bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const override { CExtKey extkey; @@ -449,7 +523,7 @@ public: return false; } - bool ToStringHelper(const SigningProvider* arg, std::string& out, bool priv) const + bool ToStringHelper(const SigningProvider* arg, std::string& out, bool priv, bool normalized) const { std::string extra = ToStringExtra(); size_t pos = extra.size() > 0 ? 1 : 0; @@ -457,7 +531,9 @@ public: for (const auto& pubkey : m_pubkey_args) { if (pos++) ret += ","; std::string tmp; - if (priv) { + if (normalized) { + if (!pubkey->ToNormalizedString(*arg, tmp, priv)) return false; + } else if (priv) { if (!pubkey->ToPrivateString(*arg, tmp)) return false; } else { tmp = pubkey->ToString(); @@ -467,7 +543,7 @@ public: if (m_subdescriptor_arg) { if (pos++) ret += ","; std::string tmp; - if (!m_subdescriptor_arg->ToStringHelper(arg, tmp, priv)) return false; + if (!m_subdescriptor_arg->ToStringHelper(arg, tmp, priv, normalized)) return false; ret += std::move(tmp); } out = std::move(ret) + ")"; @@ -477,13 +553,20 @@ public: std::string ToString() const final { std::string ret; - ToStringHelper(nullptr, ret, false); + ToStringHelper(nullptr, ret, false, false); return AddChecksum(ret); } bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { - bool ret = ToStringHelper(&arg, out, true); + bool ret = ToStringHelper(&arg, out, true, false); + out = AddChecksum(out); + return ret; + } + + bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const override final + { + bool ret = ToStringHelper(&arg, out, priv, true); out = AddChecksum(out); return ret; } @@ -565,7 +648,7 @@ public: Optional<OutputType> GetOutputType() const override { - switch (m_destination.which()) { + switch (m_destination.index()) { case 1 /* PKHash */: case 2 /* ScriptHash */: return OutputType::LEGACY; case 3 /* WitnessV0ScriptHash */: @@ -593,7 +676,7 @@ public: { CTxDestination dest; ExtractDestination(m_script, dest); - switch (dest.which()) { + switch (dest.index()) { case 1 /* PKHash */: case 2 /* ScriptHash */: return OutputType::LEGACY; case 3 /* WitnessV0ScriptHash */: diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 17b43e7c81..46d51fa587 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -93,6 +93,9 @@ struct Descriptor { /** Convert the descriptor to a private string. This fails if the provided provider does not have the relevant private keys. */ virtual bool ToPrivateString(const SigningProvider& provider, std::string& out) const = 0; + /** Convert the descriptor to a normalized string. Normalized descriptors have the xpub at the last hardened step. This fails if the provided provider does not have the private keys to derive that xpub. */ + virtual bool ToNormalizedString(const SigningProvider& provider, std::string& out, bool priv) const = 0; + /** Expand a descriptor at a specified position. * * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 2bfe206597..c6d898a25a 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -11,7 +11,11 @@ #include <util/system.h> #include <cuckoocache.h> -#include <boost/thread/shared_mutex.hpp> + +#include <algorithm> +#include <mutex> +#include <shared_mutex> +#include <vector> namespace { /** @@ -27,7 +31,7 @@ private: CSHA256 m_salted_hasher_schnorr; typedef CuckooCache::cache<uint256, SignatureCacheHasher> map_type; map_type setValid; - boost::shared_mutex cs_sigcache; + std::shared_mutex cs_sigcache; public: CSignatureCache() @@ -62,13 +66,13 @@ public: bool Get(const uint256& entry, const bool erase) { - boost::shared_lock<boost::shared_mutex> lock(cs_sigcache); + std::shared_lock<std::shared_mutex> lock(cs_sigcache); return setValid.contains(entry, erase); } void Set(const uint256& entry) { - boost::unique_lock<boost::shared_mutex> lock(cs_sigcache); + std::unique_lock<std::shared_mutex> lock(cs_sigcache); setValid.insert(entry); } uint32_t setup_bytes(size_t n) diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 512f61f2bf..bf0ba38c2d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -8,6 +8,7 @@ #include <script/interpreter.h> #include <span.h> +#include <util/hasher.h> #include <vector> @@ -20,27 +21,6 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 8afbe9ebed..dba5ce621a 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -106,8 +106,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator std::vector<valtype> vSolutions; whichTypeRet = Solver(scriptPubKey, vSolutions); - switch (whichTypeRet) - { + switch (whichTypeRet) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -173,10 +172,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator // Could not find witnessScript, add to missing sigdata.missing_witness_script = uint256(vSolutions[0]); return false; - - default: - return false; - } + } // no default case, so the compiler can warn about missing cases + assert(false); } static CScript PushAll(const std::vector<valtype>& values) diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index 26d081f853..9781ec32af 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -179,18 +179,18 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& { // Only supports destinations which map to single public keys, i.e. P2PKH, // P2WPKH, and P2SH-P2WPKH. - if (auto id = boost::get<PKHash>(&dest)) { + if (auto id = std::get_if<PKHash>(&dest)) { return ToKeyID(*id); } - if (auto witness_id = boost::get<WitnessV0KeyHash>(&dest)) { + if (auto witness_id = std::get_if<WitnessV0KeyHash>(&dest)) { return ToKeyID(*witness_id); } - if (auto script_hash = boost::get<ScriptHash>(&dest)) { + if (auto script_hash = std::get_if<ScriptHash>(&dest)) { CScript script; CScriptID script_id(*script_hash); CTxDestination inner_dest; if (store.GetCScript(script_id, script) && ExtractDestination(script, inner_dest)) { - if (auto inner_witness_id = boost::get<WitnessV0KeyHash>(&inner_dest)) { + if (auto inner_witness_id = std::get_if<WitnessV0KeyHash>(&inner_dest)) { return ToKeyID(*inner_witness_id); } } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 57d68c7ce9..4d882cd1f1 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -45,8 +45,7 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) std::string GetTxnOutputType(TxoutType t) { - switch (t) - { + switch (t) { case TxoutType::NONSTANDARD: return "nonstandard"; case TxoutType::PUBKEY: return "pubkey"; case TxoutType::PUBKEYHASH: return "pubkeyhash"; @@ -182,7 +181,8 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) std::vector<valtype> vSolutions; TxoutType whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TxoutType::PUBKEY) { + switch (whichType) { + case TxoutType::PUBKEY: { CPubKey pubKey(vSolutions[0]); if (!pubKey.IsValid()) return false; @@ -190,26 +190,28 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = PKHash(pubKey); return true; } - else if (whichType == TxoutType::PUBKEYHASH) - { + case TxoutType::PUBKEYHASH: { addressRet = PKHash(uint160(vSolutions[0])); return true; } - else if (whichType == TxoutType::SCRIPTHASH) - { + case TxoutType::SCRIPTHASH: { addressRet = ScriptHash(uint160(vSolutions[0])); return true; - } else if (whichType == TxoutType::WITNESS_V0_KEYHASH) { + } + case TxoutType::WITNESS_V0_KEYHASH: { WitnessV0KeyHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { + } + case TxoutType::WITNESS_V0_SCRIPTHASH: { WitnessV0ScriptHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_UNKNOWN || whichType == TxoutType::WITNESS_V1_TAPROOT) { + } + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V1_TAPROOT: { WitnessUnknown unk; unk.version = vSolutions[0][0]; std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); @@ -217,8 +219,13 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = unk; return true; } - // Multisig txns have more than one address... - return false; + case TxoutType::MULTISIG: + // Multisig txns have more than one address... + case TxoutType::NULL_DATA: + case TxoutType::NONSTANDARD: + return false; + } // no default case, so the compiler can warn about missing cases + assert(false); } bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) @@ -261,9 +268,8 @@ bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::v return true; } -namespace -{ -class CScriptVisitor : public boost::static_visitor<CScript> +namespace { +class CScriptVisitor { public: CScript operator()(const CNoDestination& dest) const @@ -300,7 +306,7 @@ public: CScript GetScriptForDestination(const CTxDestination& dest) { - return boost::apply_visitor(CScriptVisitor(), dest); + return std::visit(CScriptVisitor(), dest); } CScript GetScriptForRawPubKey(const CPubKey& pubKey) @@ -320,5 +326,5 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) } bool IsValidDestination(const CTxDestination& dest) { - return dest.which() != 0; + return dest.index() != 0; } diff --git a/src/script/standard.h b/src/script/standard.h index 4d1ef61964..d5d87392ad 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -9,10 +9,8 @@ #include <script/interpreter.h> #include <uint256.h> -#include <boost/variant.hpp> - #include <string> - +#include <variant> static const bool DEFAULT_ACCEPT_DATACARRIER = true; @@ -211,7 +209,7 @@ struct WitnessUnknown * (taproot outputs do not require their own type as long as no wallet support exists) * A CTxDestination is the internal data type encoded in a bitcoin address */ -typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown> CTxDestination; +using CTxDestination = std::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown>; /** Check whether a CTxDestination is a CNoDestination. */ bool IsValidDestination(const CTxDestination& dest); diff --git a/src/shutdown.cpp b/src/shutdown.cpp index df5f996022..6a73e0b2d8 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -32,7 +32,7 @@ static int g_shutdown_pipe[2] = {-1, -1}; bool InitShutdownState() { #ifndef WIN32 -#if HAVE_O_CLOEXEC +#if HAVE_O_CLOEXEC && HAVE_DECL_PIPE2 // If we can, make sure that the file descriptors are closed on exec() // to prevent interference. if (pipe2(g_shutdown_pipe, O_CLOEXEC) != 0) { diff --git a/src/streams.h b/src/streams.h index fce411d3df..e78da31cbc 100644 --- a/src/streams.h +++ b/src/streams.h @@ -6,17 +6,19 @@ #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H -#include <support/allocators/zeroafterfree.h> #include <serialize.h> +#include <span.h> +#include <support/allocators/zeroafterfree.h> #include <algorithm> #include <assert.h> #include <ios> #include <limits> +#include <optional> #include <stdint.h> #include <stdio.h> -#include <string> #include <string.h> +#include <string> #include <utility> #include <vector> @@ -202,14 +204,14 @@ public: class CDataStream { protected: - typedef CSerializeData vector_type; + using vector_type = SerializeData; vector_type vch; - unsigned int nReadPos; + unsigned int nReadPos{0}; int nType; int nVersion; -public: +public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; typedef vector_type::difference_type difference_type; @@ -221,62 +223,22 @@ public: typedef vector_type::reverse_iterator reverse_iterator; explicit CDataStream(int nTypeIn, int nVersionIn) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + : nType{nTypeIn}, + nVersion{nVersionIn} {} - CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + explicit CDataStream(Span<const uint8_t> sp, int nTypeIn, int nVersionIn) + : vch(sp.data(), sp.data() + sp.size()), + nType{nTypeIn}, + nVersion{nVersionIn} {} template <typename... Args> CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + : nType{nTypeIn}, + nVersion{nVersionIn} { - Init(nTypeIn, nVersionIn); ::SerializeMany(*this, std::forward<Args>(args)...); } - void Init(int nTypeIn, int nVersionIn) - { - nReadPos = 0; - nType = nTypeIn; - nVersion = nVersionIn; - } - - CDataStream& operator+=(const CDataStream& b) - { - vch.insert(vch.end(), b.begin(), b.end()); - return *this; - } - - friend CDataStream operator+(const CDataStream& a, const CDataStream& b) - { - CDataStream ret = a; - ret += b; - return (ret); - } - std::string str() const { return (std::string(begin(), end())); @@ -297,12 +259,12 @@ public: const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } + iterator insert(iterator it, const uint8_t x) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const uint8_t x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } - void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) + void insert(iterator it, std::vector<uint8_t>::const_iterator first, std::vector<uint8_t>::const_iterator last) { if (last == first) return; assert(last - first > 0); @@ -373,12 +335,17 @@ public: nReadPos = 0; } - bool Rewind(size_type n) + bool Rewind(std::optional<size_type> n = std::nullopt) { + // Total rewind if no size is passed + if (!n) { + nReadPos = 0; + return true; + } // Rewind by n characters if the buffer hasn't been compacted yet - if (n > nReadPos) + if (*n > nReadPos) return false; - nReadPos -= n; + nReadPos -= *n; return true; } @@ -462,11 +429,6 @@ public: return (*this); } - void GetAndClear(CSerializeData &d) { - d.insert(d.end(), begin(), end()); - clear(); - } - /** * XOR the contents of this stream with a certain key. * diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index c7ed5ef308..418f0ee656 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -42,7 +42,7 @@ struct zero_after_free_allocator : public std::allocator<T> { } }; -// Byte-vector that clears its contents before deletion. -typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; +/** Byte-vector that clears its contents before deletion. */ +using SerializeData = std::vector<uint8_t, zero_after_free_allocator<uint8_t>>; #endif // BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H diff --git a/src/sync.cpp b/src/sync.cpp index acfbe8fe29..a2b62c2286 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -13,8 +13,6 @@ #include <util/strencodings.h> #include <util/threadnames.h> -#include <boost/thread/mutex.hpp> - #include <map> #include <mutex> #include <set> @@ -224,7 +222,6 @@ template void EnterCritical(const char*, const char*, int, Mutex*, bool); template void EnterCritical(const char*, const char*, int, RecursiveMutex*, bool); template void EnterCritical(const char*, const char*, int, std::mutex*, bool); template void EnterCritical(const char*, const char*, int, std::recursive_mutex*, bool); -template void EnterCritical(const char*, const char*, int, boost::mutex*, bool); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { diff --git a/src/sync.h b/src/sync.h index 749bf5575c..53213c2089 100644 --- a/src/sync.h +++ b/src/sync.h @@ -258,7 +258,22 @@ using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove //! //! int val = WITH_LOCK(cs, return shared_val); //! -#define WITH_LOCK(cs, code) [&] { LOCK(cs); code; }() +//! Note: +//! +//! Since the return type deduction follows that of decltype(auto), while the +//! deduced type of: +//! +//! WITH_LOCK(cs, return {int i = 1; return i;}); +//! +//! is int, the deduced type of: +//! +//! WITH_LOCK(cs, return {int j = 1; return (j);}); +//! +//! is &int, a reference to a local variable +//! +//! The above is detectable at compile-time with the -Wreturn-local-addr flag in +//! gcc and the -Wreturn-stack-address flag in clang, both enabled by default. +#define WITH_LOCK(cs, code) [&]() -> decltype(auto) { LOCK(cs); code; }() class CSemaphore { diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 3f7c5e99ee..3b44564ddb 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) static const std::string vstrIn[] = {"","f","fo","foo","foob","fooba","foobar"}; static const std::string vstrOut[] = {"","my======","mzxq====","mzxw6===","mzxw6yq=","mzxw6ytb","mzxw6ytboi======"}; static const std::string vstrOutNoPadding[] = {"","my","mzxq","mzxw6","mzxw6yq","mzxw6ytb","mzxw6ytboi"}; - for (unsigned int i=0; i<sizeof(vstrIn)/sizeof(vstrIn[0]); i++) + for (unsigned int i=0; i<std::size(vstrIn); i++) { std::string strEnc = EncodeBase32(vstrIn[i]); BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index bb8d102bd0..714fccffaa 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -16,7 +16,7 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) { static const std::string vstrIn[] = {"","f","fo","foo","foob","fooba","foobar"}; static const std::string vstrOut[] = {"","Zg==","Zm8=","Zm9v","Zm9vYg==","Zm9vYmE=","Zm9vYmFy"}; - for (unsigned int i=0; i<sizeof(vstrIn)/sizeof(vstrIn[0]); i++) + for (unsigned int i=0; i<std::size(vstrIn); i++) { std::string strEnc = EncodeBase64(vstrIn[i]); BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 00c4bdc14e..633a95ce96 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -178,7 +178,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -196,7 +196,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -210,7 +210,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -231,14 +231,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { LOCK(cs_main); - block_index = LookupBlockIndex(chainA[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainA_last_header); { LOCK(cs_main); - block_index = LookupBlockIndex(chainB[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainB_last_header); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 736c260eeb..5a98558240 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -42,11 +42,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03614e9b050000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03614e9b050000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); @@ -72,11 +68,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03ce4299050000000100008001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03ce4299050000000100008001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -96,11 +88,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("038fc16b080000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<unsigned char> expected = ParseHex("038fc16b080000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -352,11 +340,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) CDataStream merkleStream(SER_NETWORK, PROTOCOL_VERSION); merkleStream << merkleBlock; - std::vector<unsigned char> vch = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), merkleStream.begin(), merkleStream.end()); } diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 5dbf07b420..21921375b3 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -10,7 +10,6 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> #include <atomic> #include <condition_variable> @@ -148,10 +147,7 @@ typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; static void Correct_Queue_range(std::vector<size_t> range) { auto small_queue = MakeUnique<Correct_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{small_queue->Thread();}); - } + small_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); // Make vChecks here to save on malloc (this test can be slow...) std::vector<FakeCheckCheckCompletion> vChecks; for (const size_t i : range) { @@ -168,8 +164,7 @@ static void Correct_Queue_range(std::vector<size_t> range) BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); } } - tg.interrupt_all(); - tg.join_all(); + small_queue->StopWorkerThreads(); } /** Test that 0 checks is correct @@ -212,11 +207,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1001; ++i) { CCheckQueueControl<FailingCheck> control(fail_queue.get()); @@ -237,18 +228,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) BOOST_REQUIRE(success); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that a block validation which fails does not interfere with // future blocks, ie, the bad state is cleared. BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (auto times = 0; times < 10; ++times) { for (const bool end_fails : {true, false}) { @@ -263,8 +250,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_REQUIRE(r != end_fails); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that unique checks are actually all called individually, rather than @@ -273,11 +259,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { auto queue = MakeUnique<Unique_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); size_t COUNT = 100000; size_t total = COUNT; @@ -300,8 +282,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) } BOOST_REQUIRE(r); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } @@ -313,10 +294,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { auto queue = MakeUnique<Memory_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1000; ++i) { size_t total = i; { @@ -335,8 +313,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) } BOOST_REQUIRE_EQUAL(MemoryCheck::fake_allocated_memory, 0U); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } // Test that a new verification cannot occur until all checks @@ -344,11 +321,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { auto queue = MakeUnique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; bool fails = false; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); std::thread t0([&]() { CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); std::vector<FrozenCleanupCheck> vChecks(1); @@ -367,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) } // Try to get control of the queue a bunch of times for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } { // Unfreeze (we need lock n case of spurious wakeup) @@ -378,9 +352,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) FrozenCleanupCheck::cv.notify_one(); // Wait for control to finish t0.join(); - tg.interrupt_all(); - tg.join_all(); BOOST_REQUIRE(!fails); + queue->StopWorkerThreads(); } @@ -389,11 +362,11 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) { auto queue = MakeUnique<Standard_Queue>(QUEUE_BATCH_SIZE); { - boost::thread_group tg; + std::vector<std::thread> tg; std::atomic<int> nThreads {0}; std::atomic<int> fails {0}; for (size_t i = 0; i < 3; ++i) { - tg.create_thread( + tg.emplace_back( [&]{ CCheckQueueControl<FakeCheck> control(queue.get()); // While sleeping, no other thread should execute to this point @@ -402,11 +375,13 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) fails += observed != nThreads; }); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } BOOST_REQUIRE_EQUAL(fails, 0); } { - boost::thread_group tg; + std::vector<std::thread> tg; std::mutex m; std::condition_variable cv; bool has_lock{false}; @@ -415,7 +390,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) bool done_ack{false}; { std::unique_lock<std::mutex> l(m); - tg.create_thread([&]{ + tg.emplace_back([&]{ CCheckQueueControl<FakeCheck> control(queue.get()); std::unique_lock<std::mutex> ll(m); has_lock = true; @@ -431,7 +406,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.wait(l, [&](){return has_lock;}); bool fails = false; for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } has_tried = true; cv.notify_one(); @@ -441,8 +416,9 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.notify_one(); BOOST_REQUIRE(!fails); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } } } BOOST_AUTO_TEST_SUITE_END() - diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 3a951d28ae..35b66cfc53 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,13 +1,18 @@ // Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <boost/test/unit_test.hpp> #include <cuckoocache.h> -#include <deque> #include <random.h> #include <script/sigcache.h> #include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <deque> +#include <mutex> +#include <shared_mutex> #include <thread> +#include <vector> /** Test Suite for CuckooCache * @@ -199,11 +204,11 @@ static void test_cache_erase_parallel(size_t megabytes) * "future proofed". */ std::vector<uint256> hashes_insert_copy = hashes; - boost::shared_mutex mtx; + std::shared_mutex mtx; { /** Grab lock to make sure we release inserts */ - boost::unique_lock<boost::shared_mutex> l(mtx); + std::unique_lock<std::shared_mutex> l(mtx); /** Insert the first half */ for (uint32_t i = 0; i < (n_insert / 2); ++i) set.insert(hashes_insert_copy[i]); @@ -217,7 +222,7 @@ static void test_cache_erase_parallel(size_t megabytes) /** Each thread is emplaced with x copy-by-value */ threads.emplace_back([&, x] { - boost::shared_lock<boost::shared_mutex> l(mtx); + std::shared_lock<std::shared_mutex> l(mtx); size_t ntodo = (n_insert/4)/3; size_t start = ntodo*x; size_t end = ntodo*(x+1); @@ -232,7 +237,7 @@ static void test_cache_erase_parallel(size_t megabytes) for (std::thread& t : threads) t.join(); /** Grab lock to make sure we observe erases */ - boost::unique_lock<boost::shared_mutex> l(mtx); + std::unique_lock<std::shared_mutex> l(mtx); /** Insert the second half */ for (uint32_t i = (n_insert / 2); i < n_insert; ++i) set.insert(hashes_insert_copy[i]); diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index 3b1db449b2..c394356798 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -3,27 +3,28 @@ ["They are in the form"], ["[[[prevout hash, prevout index, prevout scriptPubKey, amount?], [input 2], ...],"], ["serializedTransaction, verifyFlags]"], +["Use BADTX for verifyFlags if it is expected to fail CheckTransaction()"], ["Objects that are only a single string (like this one) are ignored"], ["0e1b5688cf179cd9f7cbda1fac0090f6e684bbf8cd946660120197c3f3681809 but with extra junk appended to the end of the scriptPubKey"], [[["6ca7ec7b1847f6bdbd737176050e6a08d66ccd55bb94ad24f4018024107a5827", 0, "0x41 0x043b640e983c9690a14c039a2037ecc3467b27a0dcd58f19d76c7bc118d09fec45adc5370a1c5bf8067ca9f5557a4cf885fdb0fe0dcc9c3a7137226106fbc779a5 CHECKSIG VERIFY 1"]], -"010000000127587a10248001f424ad94bb55cd6cd6086a0e05767173bdbdf647187beca76c000000004948304502201b822ad10d6adc1a341ae8835be3f70a25201bbff31f59cbb9c5353a5f0eca18022100ea7b2f7074e9aa9cf70aa8d0ffee13e6b45dddabf1ab961bda378bcdb778fa4701ffffffff0100f2052a010000001976a914fc50c5907d86fed474ba5ce8b12a66e0a4c139d888ac00000000", "P2SH"], +"010000000127587a10248001f424ad94bb55cd6cd6086a0e05767173bdbdf647187beca76c000000004948304502201b822ad10d6adc1a341ae8835be3f70a25201bbff31f59cbb9c5353a5f0eca18022100ea7b2f7074e9aa9cf70aa8d0ffee13e6b45dddabf1ab961bda378bcdb778fa4701ffffffff0100f2052a010000001976a914fc50c5907d86fed474ba5ce8b12a66e0a4c139d888ac00000000", "NONE"], ["This is the nearly-standard transaction with CHECKSIGVERIFY 1 instead of CHECKSIG from tx_valid.json"], ["but with the signature duplicated in the scriptPubKey with a non-standard pushdata prefix"], ["See FindAndDelete, which will only remove if it uses the same pushdata prefix as is standard"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x4c 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a01"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "NONE"], ["Same as above, but with the sig in the scriptSig also pushed with the same non-standard OP_PUSHDATA"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x4c 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a01"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006b4c473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006b4c473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "NONE"], ["This is the nearly-standard transaction with CHECKSIGVERIFY 1 instead of CHECKSIG from tx_valid.json"], ["but with the signature duplicated in the scriptPubKey with a different hashtype suffix"], ["See FindAndDelete, which will only remove if the signature, including the hash type, matches"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a81"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "NONE"], ["An invalid P2SH Transaction"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]], @@ -32,46 +33,46 @@ ["Tests for CheckTransaction()"], ["No outputs"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x05ab9e14d983742513f0f451e105ffb4198d1dd4 EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022100f16703104aab4e4088317c862daec83440242411b039d14280e03dd33b487ab802201318a7be236672c5c56083eb7a5a195bc57a40af7923ff8545016cd3b571e2a601232103c40e5d339df3f30bf753e7e04450ae4ef76c9e45587d1d993bdc4cd06f0651c7acffffffff0000000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022100f16703104aab4e4088317c862daec83440242411b039d14280e03dd33b487ab802201318a7be236672c5c56083eb7a5a195bc57a40af7923ff8545016cd3b571e2a601232103c40e5d339df3f30bf753e7e04450ae4ef76c9e45587d1d993bdc4cd06f0651c7acffffffff0000000000", "BADTX"], ["Negative output"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xae609aca8061d77c5e111f6bb62501a6bbe2bfdb EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d4830450220063222cbb128731fc09de0d7323746539166544d6c1df84d867ccea84bcc8903022100bf568e8552844de664cd41648a031554327aa8844af34b4f27397c65b92c04de0123210243ec37dee0e2e053a9c976f43147e79bc7d9dc606ea51010af1ac80db6b069e1acffffffff01ffffffffffffffff015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d4830450220063222cbb128731fc09de0d7323746539166544d6c1df84d867ccea84bcc8903022100bf568e8552844de664cd41648a031554327aa8844af34b4f27397c65b92c04de0123210243ec37dee0e2e053a9c976f43147e79bc7d9dc606ea51010af1ac80db6b069e1acffffffff01ffffffffffffffff015100000000", "BADTX"], ["MAX_MONEY + 1 output"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x32afac281462b822adbec5094b8d4d337dd5bd6a EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100e1eadba00d9296c743cb6ecc703fd9ddc9b3cd12906176a226ae4c18d6b00796022100a71aef7d2874deff681ba6080f1b278bac7bb99c61b08a85f4311970ffe7f63f012321030c0588dc44d92bdcbf8e72093466766fdc265ead8db64517b0c542275b70fffbacffffffff010140075af0750700015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100e1eadba00d9296c743cb6ecc703fd9ddc9b3cd12906176a226ae4c18d6b00796022100a71aef7d2874deff681ba6080f1b278bac7bb99c61b08a85f4311970ffe7f63f012321030c0588dc44d92bdcbf8e72093466766fdc265ead8db64517b0c542275b70fffbacffffffff010140075af0750700015100000000", "BADTX"], ["MAX_MONEY output + 1 output"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xb558cbf4930954aa6a344363a15668d7477ae716 EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022027deccc14aa6668e78a8c9da3484fbcd4f9dcc9bb7d1b85146314b21b9ae4d86022100d0b43dece8cfb07348de0ca8bc5b86276fa88f7f2138381128b7c36ab2e42264012321029bb13463ddd5d2cc05da6e84e37536cb9525703cfd8f43afdb414988987a92f6acffffffff020040075af075070001510001000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022027deccc14aa6668e78a8c9da3484fbcd4f9dcc9bb7d1b85146314b21b9ae4d86022100d0b43dece8cfb07348de0ca8bc5b86276fa88f7f2138381128b7c36ab2e42264012321029bb13463ddd5d2cc05da6e84e37536cb9525703cfd8f43afdb414988987a92f6acffffffff020040075af075070001510001000000000000015100000000", "BADTX"], ["Duplicate inputs"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x236d0639db62b0773fd8ac34dc85ae19e9aba80a EQUAL"]], -"01000000020001000000000000000000000000000000000000000000000000000000000000000000006c47304402204bb1197053d0d7799bf1b30cd503c44b58d6240cccbdc85b6fe76d087980208f02204beeed78200178ffc6c74237bb74b3f276bbb4098b5605d814304fe128bf1431012321039e8815e15952a7c3fada1905f8cf55419837133bd7756c0ef14fc8dfe50c0deaacffffffff0001000000000000000000000000000000000000000000000000000000000000000000006c47304402202306489afef52a6f62e90bf750bbcdf40c06f5c6b138286e6b6b86176bb9341802200dba98486ea68380f47ebb19a7df173b99e6bc9c681d6ccf3bde31465d1f16b3012321039e8815e15952a7c3fada1905f8cf55419837133bd7756c0ef14fc8dfe50c0deaacffffffff010000000000000000015100000000", "P2SH"], +"01000000020001000000000000000000000000000000000000000000000000000000000000000000006c47304402204bb1197053d0d7799bf1b30cd503c44b58d6240cccbdc85b6fe76d087980208f02204beeed78200178ffc6c74237bb74b3f276bbb4098b5605d814304fe128bf1431012321039e8815e15952a7c3fada1905f8cf55419837133bd7756c0ef14fc8dfe50c0deaacffffffff0001000000000000000000000000000000000000000000000000000000000000000000006c47304402202306489afef52a6f62e90bf750bbcdf40c06f5c6b138286e6b6b86176bb9341802200dba98486ea68380f47ebb19a7df173b99e6bc9c681d6ccf3bde31465d1f16b3012321039e8815e15952a7c3fada1905f8cf55419837133bd7756c0ef14fc8dfe50c0deaacffffffff010000000000000000015100000000", "BADTX"], ["Coinbase of size 1"], ["Note the input is just required to make the tester happy"], [[["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"]], -"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0151ffffffff010000000000000000015100000000", "P2SH"], +"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0151ffffffff010000000000000000015100000000", "BADTX"], ["Coinbase of size 101"], ["Note the input is just required to make the tester happy"], [[["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"]], -"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff655151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151ffffffff010000000000000000015100000000", "P2SH"], +"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff655151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151ffffffff010000000000000000015100000000", "BADTX"], ["Null txin, but without being a coinbase (because there are two inputs)"], [[["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"01000000020000000000000000000000000000000000000000000000000000000000000000ffffffff00ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015100000000", "P2SH"], +"01000000020000000000000000000000000000000000000000000000000000000000000000ffffffff00ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015100000000", "BADTX"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"], ["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"]], -"010000000200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0000000000000000000000000000000000000000000000000000000000000000ffffffff00ffffffff010000000000000000015100000000", "P2SH"], +"010000000200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0000000000000000000000000000000000000000000000000000000000000000ffffffff00ffffffff010000000000000000015100000000", "BADTX"], ["Same as the transactions in valid with one input SIGHASH_ALL and one SIGHASH_ANYONECANPAY, but we set the _ANYONECANPAY sequence number, invalidating the SIGHASH_ALL signature"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"], ["0000000000000000000000000000000000000000000000000000000000000200", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"]], - "01000000020001000000000000000000000000000000000000000000000000000000000000000000004948304502203a0f5f0e1f2bdbcd04db3061d18f3af70e07f4f467cbc1b8116f267025f5360b022100c792b6e215afc5afc721a351ec413e714305cb749aae3d7fee76621313418df10101000000000200000000000000000000000000000000000000000000000000000000000000000000484730440220201dc2d030e380e8f9cfb41b442d930fa5a685bb2c8db5906671f865507d0670022018d9e7a8d4c8d86a73c2a724ee38ef983ec249827e0e464841735955c707ece98101000000010100000000000000015100000000", "P2SH"], + "01000000020001000000000000000000000000000000000000000000000000000000000000000000004948304502203a0f5f0e1f2bdbcd04db3061d18f3af70e07f4f467cbc1b8116f267025f5360b022100c792b6e215afc5afc721a351ec413e714305cb749aae3d7fee76621313418df10101000000000200000000000000000000000000000000000000000000000000000000000000000000484730440220201dc2d030e380e8f9cfb41b442d930fa5a685bb2c8db5906671f865507d0670022018d9e7a8d4c8d86a73c2a724ee38ef983ec249827e0e464841735955c707ece98101000000010100000000000000015100000000", "NONE"], ["CHECKMULTISIG with incorrect signature order"], ["Note the input is just required to make the tester happy"], @@ -82,7 +83,7 @@ ["The following is a tweaked form of 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63"], ["It is an OP_CHECKMULTISIG with the dummy value missing"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004847304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004847304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "NONE"], ["CHECKMULTISIG SCRIPT_VERIFY_NULLDUMMY tests:"], @@ -90,104 +91,104 @@ ["The following is a tweaked form of 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63"], ["It is an OP_CHECKMULTISIG with the dummy value set to something other than an empty string"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a010047304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH,NULLDUMMY"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a010047304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "NULLDUMMY"], ["As above, but using an OP_1"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000495147304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH,NULLDUMMY"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000495147304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "NULLDUMMY"], ["As above, but using an OP_1NEGATE"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000494f47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH,NULLDUMMY"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000494f47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "NULLDUMMY"], ["As above, but with the dummy byte missing"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004847304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH,NULLDUMMY"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004847304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "NONE"], ["Empty stack when we try to run CHECKSIG"], [[["ad503f72c18df5801ee64d76090afe4c607fb2b822e9b7b63c5826c50e22fc3b", 0, "0x21 0x027c3a97665bf283a102a587a62a30a0c102d4d3b141015e2cae6f64e2543113e5 CHECKSIG NOT"]], -"01000000013bfc220ec526583cb6b7e922b8b27f604cfe0a09764de61e80f58dc1723f50ad0000000000ffffffff0101000000000000002321027c3a97665bf283a102a587a62a30a0c102d4d3b141015e2cae6f64e2543113e5ac00000000", "P2SH"], +"01000000013bfc220ec526583cb6b7e922b8b27f604cfe0a09764de61e80f58dc1723f50ad0000000000ffffffff0101000000000000002321027c3a97665bf283a102a587a62a30a0c102d4d3b141015e2cae6f64e2543113e5ac00000000", "NONE"], ["Inverted versions of tx_valid CODESEPARATOR IF block tests"], ["CODESEPARATOR in an unexecuted IF block does not change what is hashed"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], -"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0151ffffffff010000000000000000016a00000000", "P2SH"], +"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0151ffffffff010000000000000000016a00000000", "NONE"], ["As above, with the IF block executed"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], -"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510100ffffffff010000000000000000016a00000000", "P2SH"], +"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510100ffffffff010000000000000000016a00000000", "NONE"], ["CHECKLOCKTIMEVERIFY tests"], ["By-height locks, with argument just beyond tx nLockTime"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000fe64cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000fe64cd1d", "CHECKLOCKTIMEVERIFY"], ["By-time locks, with argument just beyond tx nLockTime (but within numerical boundaries)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000001 CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000feffffff", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000feffffff", "CHECKLOCKTIMEVERIFY"], ["Argument missing"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000001b1010000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000001b1010000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], ["Argument negative with by-blockheight nLockTime=0"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "-1 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], ["Argument negative with by-blocktime nLockTime=500,000,000"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "-1 CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000004005194b1010000000100000000000000000002000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000004005194b1010000000100000000000000000002000000", "CHECKLOCKTIMEVERIFY"], ["Input locked"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1ffffffff0100000000000000000002000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1ffffffff0100000000000000000002000000", "NONE"], ["Another input being unlocked isn't sufficient; the CHECKLOCKTIMEVERIFY-using input must be unlocked"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"] , ["0000000000000000000000000000000000000000000000000000000000000200", 1, "1"]], -"010000000200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00020000000000000000000000000000000000000000000000000000000000000100000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00020000000000000000000000000000000000000000000000000000000000000100000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], ["Argument/tx height/time mismatch, both versions"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b100000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b100000000010000000000000000000065cd1d", "NONE"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "CHECKLOCKTIMEVERIFY"], ["Argument 2^32 with nLockTime=2^32-1"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000000001 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "CHECKLOCKTIMEVERIFY"], ["Same, but with nLockTime=2^31-1"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffff7f", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffff7f", "CHECKLOCKTIMEVERIFY"], ["6 byte non-minimally-encoded arguments are invalid even if their contents are valid"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x06 0x000000000000 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], ["Failure due to failing CHECKLOCKTIMEVERIFY in scriptSig"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1000000000100000000000000000000000000", "CHECKLOCKTIMEVERIFY"], ["Failure due to failing CHECKLOCKTIMEVERIFY in redeemScript"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xc5b93064159b3b2d6ab506a41b1f50463771b988 EQUAL"]], @@ -195,51 +196,51 @@ ["A transaction with a non-standard DER signature."], [[["b1dbc81696c8a9c0fccd0693ab66d7c368dbc38c0def4e800685560ddd1b2132", 0, "DUP HASH160 0x14 0x4b3bd7eba3bc0284fd3007be7f3be275e94f5826 EQUALVERIFY CHECKSIG"]], -"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "P2SH,DERSIG"], +"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "DERSIG"], ["CHECKSEQUENCEVERIFY tests"], ["By-height locks, with argument just beyond txin.nSequence"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feff40000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feff40000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["By-time locks, with argument just beyond txin.nSequence (but within numerical boundaries)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194305 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feff40000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feff40000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Argument missing"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Argument negative with by-blockheight txin.nSequence=0"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "-1 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Argument negative with by-blocktime txin.nSequence=CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "-1 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Argument/tx height/time mismatch, both versions"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "65535 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["6 byte non-minimally-encoded arguments are invalid even if their contents are valid"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x06 0x000000000000 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Failure due to failing CHECKSEQUENCEVERIFY in scriptSig"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"02000000010001000000000000000000000000000000000000000000000000000000000000000000000251b2000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"02000000010001000000000000000000000000000000000000000000000000000000000000000000000251b2000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Failure due to failing CHECKSEQUENCEVERIFY in redeemScript"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7c17aff532f22beb54069942f9bf567a66133eaf EQUAL"]], @@ -247,9 +248,9 @@ ["Failure due to insufficient tx.nVersion (<2)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "CHECKSEQUENCEVERIFY"], ["Unknown witness program version (with DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], @@ -273,7 +274,7 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff000100000000000000000000000000000000000000000000000000000000000001000000000100000000010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff000100000000000000000000000000000000000000000000000000000000000001000000000100000000010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], ["Witness with SigHash All|AnyoneCanPay (third output value changed)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], @@ -287,7 +288,7 @@ ["Witness with unknown version which push false on the stack should be invalid (even without DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x02 0x0000", 2000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015101010100000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015101010100000000", "NONE"], ["Witness program should leave clean stack"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x20 0x2f04a3aa051f1f60d695f6c44c0c3d383973dfd446ace8962664a76bb10e31a8", 2000]], @@ -304,11 +305,11 @@ ["Non witness Single|AnyoneCanPay hash input's position (permutation)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x21 0x03596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71 CHECKSIG", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x21 0x03596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71 CHECKSIG", 1001]], -"010000000200010000000000000000000000000000000000000000000000000000000000000100000049483045022100acb96cfdbda6dc94b489fd06f2d720983b5f350e31ba906cdbd800773e80b21c02200d74ea5bdf114212b4bbe9ed82c36d2e369e302dff57cb60d01c428f0bd3daab83ffffffff0001000000000000000000000000000000000000000000000000000000000000000000004847304402202a0b4b1294d70540235ae033d78e64b4897ec859c7b6f1b2b1d8a02e1d46006702201445e756d2254b0f1dfda9ab8e1e1bc26df9668077403204f32d16a49a36eb6983ffffffff02e9030000000000000151e803000000000000015100000000", "P2SH,WITNESS"], +"010000000200010000000000000000000000000000000000000000000000000000000000000100000049483045022100acb96cfdbda6dc94b489fd06f2d720983b5f350e31ba906cdbd800773e80b21c02200d74ea5bdf114212b4bbe9ed82c36d2e369e302dff57cb60d01c428f0bd3daab83ffffffff0001000000000000000000000000000000000000000000000000000000000000000000004847304402202a0b4b1294d70540235ae033d78e64b4897ec859c7b6f1b2b1d8a02e1d46006702201445e756d2254b0f1dfda9ab8e1e1bc26df9668077403204f32d16a49a36eb6983ffffffff02e9030000000000000151e803000000000000015100000000", "NONE"], -["P2WSH with a redeem representing a witness scriptPubKey should fail"], +["P2WSH with a redeem representing a witness scriptPubKey should fail due to too many stack items"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x20 0x34b6c399093e06cf9f0f7f660a1abcfe78fcf7b576f43993208edd9518a0ae9b", 1000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0001045102010100000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015101045102010100000000", "P2SH,WITNESS"], ["33 bytes push should be considered a witness scriptPubKey"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x21 0xff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbff", 1000]], @@ -324,7 +325,7 @@ ["by replacing (sigversion == SigVersion::BASE) with (sigversion != SigVersion::BASE)"], ["Non-segwit: wrong sighash (without FindAndDelete) = 1ba1fe3bc90c5d1265460e684ce6774e324f0fabdf67619eda729e64e8b6bc08"], [[["f18783ace138abac5d3a7a5cf08e88fe6912f267ef936452e0c27d090621c169", 7000, "HASH160 0x14 0x0c746489e2d83cdbb5b90b432773342ba809c134 EQUAL", 200000]], -"010000000169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f1581b0000b64830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e012103b12a1ec8428fc74166926318c15e17408fea82dbb157575e16a8c365f546248f4aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e01ffffffff0101000000000000000000000000", "P2SH,WITNESS"], +"010000000169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f1581b0000b64830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e012103b12a1ec8428fc74166926318c15e17408fea82dbb157575e16a8c365f546248f4aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e01ffffffff0101000000000000000000000000", "P2SH"], ["BIP143: wrong sighash (with FindAndDelete) = 71c9cd9b2869b9c70b01b1f0360c148f42dee72297db312638df136f43311f23"], [[["f18783ace138abac5d3a7a5cf08e88fe6912f267ef936452e0c27d090621c169", 7500, "0x00 0x20 0x9e1be07558ea5cc8e02ed1d80c0911048afad949affa36d5c3951e3159dbea19", 200000]], "0100000000010169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f14c1d000000ffffffff01010000000000000000034830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e012102a9d7ed6e161f0e255c10bbfcca0128a9e2035c2c8da58899c54d22d3a31afdef4aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0100000000", "P2SH,WITNESS"], @@ -335,7 +336,7 @@ ["Should pass by replacing (sigversion == SigVersion::BASE) with (sigversion != SigVersion::BASE) under OP_CHECKMULTISIG"], ["Non-segwit: wrong sighash (without FindAndDelete) = 4bc6a53e8e16ef508c19e38bba08831daba85228b0211f323d4cb0999cf2a5e8"], [[["9628667ad48219a169b41b020800162287d2c0f713c04157e95c484a8dcb7592", 7000, "HASH160 0x14 0x5748407f5ca5cdca53ba30b79040260770c9ee1b EQUAL", 200000]], -"01000000019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a662896581b0000fd6f01004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c039596015221023fd5dd42b44769c5653cbc5947ff30ab8871f240ad0c0e7432aefe84b5b4ff3421039d52178dbde360b83f19cf348deb04fa8360e1bf5634577be8e50fafc2b0e4ef4c9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960175ffffffff0101000000000000000000000000", "P2SH,WITNESS"], +"01000000019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a662896581b0000fd6f01004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c039596015221023fd5dd42b44769c5653cbc5947ff30ab8871f240ad0c0e7432aefe84b5b4ff3421039d52178dbde360b83f19cf348deb04fa8360e1bf5634577be8e50fafc2b0e4ef4c9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960175ffffffff0101000000000000000000000000", "P2SH"], ["BIP143: wrong sighash (with FindAndDelete) = 17c50ec2181ecdfdc85ca081174b248199ba81fff730794d4f69b8ec031f2dce"], [[["9628667ad48219a169b41b020800162287d2c0f713c04157e95c484a8dcb7592", 7500, "0x00 0x20 0x9b66c15b4e0b4eb49fa877982cafded24859fe5b0e2dbfbe4f0df1de7743fd52", 200000]], "010000000001019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a6628964c1d000000ffffffff0101000000000000000007004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c03959601010221023cb6055f4b57a1580c5a753e19610cafaedf7e0ff377731c77837fd666eae1712102c1b1db303ac232ffa8e5e7cc2cf5f96c6e40d3e6914061204c0541cb2043a0969552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c039596017500000000", "P2SH,WITNESS"], @@ -344,39 +345,39 @@ ["All transactions are copied from OP_CODESEPARATOR tests in tx_valid.json"], [[["bc7fd132fcf817918334822ee6d9bd95c889099c96e07ca2c1eb2cc70db63224", 0, "CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIG"]], - "01000000012432b60dc72cebc1a27ce0969c0989c895bdd9e62e8234839117f8fc32d17fbc000000004a493046022100a576b52051962c25e642c0fd3d77ee6c92487048e5d90818bcf5b51abaccd7900221008204f8fb121be4ec3b24483b1f92d89b1b0548513a134e345c5442e86e8617a501ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000012432b60dc72cebc1a27ce0969c0989c895bdd9e62e8234839117f8fc32d17fbc000000004a493046022100a576b52051962c25e642c0fd3d77ee6c92487048e5d90818bcf5b51abaccd7900221008204f8fb121be4ec3b24483b1f92d89b1b0548513a134e345c5442e86e8617a501ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], [[["83e194f90b6ef21fa2e3a365b63794fb5daa844bdc9b25de30899fcfe7b01047", 0, "CODESEPARATOR CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIG"]], - "01000000014710b0e7cf9f8930de259bdc4b84aa5dfb9437b665a3e3a21ff26e0bf994e183000000004a493046022100a166121a61b4eeb19d8f922b978ff6ab58ead8a5a5552bf9be73dc9c156873ea02210092ad9bc43ee647da4f6652c320800debcf08ec20a094a0aaf085f63ecb37a17201ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000014710b0e7cf9f8930de259bdc4b84aa5dfb9437b665a3e3a21ff26e0bf994e183000000004a493046022100a166121a61b4eeb19d8f922b978ff6ab58ead8a5a5552bf9be73dc9c156873ea02210092ad9bc43ee647da4f6652c320800debcf08ec20a094a0aaf085f63ecb37a17201ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], [[["326882a7f22b5191f1a0cc9962ca4b878cd969cf3b3a70887aece4d801a0ba5e", 0, "0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CODESEPARATOR CHECKSIG"]], - "01000000015ebaa001d8e4ec7a88703a3bcf69d98c874bca6299cca0f191512bf2a7826832000000004948304502203bf754d1c6732fbf87c5dcd81258aefd30f2060d7bd8ac4a5696f7927091dad1022100f5bcb726c4cf5ed0ed34cc13dadeedf628ae1045b7cb34421bc60b89f4cecae701ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000015ebaa001d8e4ec7a88703a3bcf69d98c874bca6299cca0f191512bf2a7826832000000004948304502203bf754d1c6732fbf87c5dcd81258aefd30f2060d7bd8ac4a5696f7927091dad1022100f5bcb726c4cf5ed0ed34cc13dadeedf628ae1045b7cb34421bc60b89f4cecae701ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIGVERIFY CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIGVERIFY CODESEPARATOR 1"]], - "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a900000000924830450221009c0a27f886a1d8cb87f6f595fbc3163d28f7a81ec3c4b252ee7f3ac77fd13ffa02203caa8dfa09713c8c4d7ef575c75ed97812072405d932bd11e6a1593a98b679370148304502201e3861ef39a526406bad1e20ecad06be7375ad40ddb582c9be42d26c3a0d7b240221009d0a3985e96522e59635d19cc4448547477396ce0ef17a58e7d74c3ef464292301ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a900000000924830450221009c0a27f886a1d8cb87f6f595fbc3163d28f7a81ec3c4b252ee7f3ac77fd13ffa02203caa8dfa09713c8c4d7ef575c75ed97812072405d932bd11e6a1593a98b679370148304502201e3861ef39a526406bad1e20ecad06be7375ad40ddb582c9be42d26c3a0d7b240221009d0a3985e96522e59635d19cc4448547477396ce0ef17a58e7d74c3ef464292301ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["CODESEPARATOR in an unexecuted IF block is still invalid"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], - "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0100ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0100ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["CODESEPARATOR in an executed IF block is invalid"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], - "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510151ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510151ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["Using CHECKSIG with signatures in scriptSigs will trigger FindAndDelete, which is invalid"], [[["ccf7f4053a02e653c36ac75c891b7496d0dc5ce5214f6c913d9cf8f1329ebee0", 0, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], - "0100000001e0be9e32f1f89c3d916c4f21e55cdcd096741b895cc76ac353e6023a05f4f7cc00000000d86149304602210086e5f736a2c3622ebb62bd9d93d8e5d76508b98be922b97160edc3dcca6d8c47022100b23c312ac232a4473f19d2aeb95ab7bdf2b65518911a0d72d50e38b5dd31dc820121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac4730440220508fa761865c8abd81244a168392876ee1d94e8ed83897066b5e2df2400dad24022043f5ee7538e87e9c6aef7ef55133d3e51da7cc522830a9c4d736977a76ef755c0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "0100000001e0be9e32f1f89c3d916c4f21e55cdcd096741b895cc76ac353e6023a05f4f7cc00000000d86149304602210086e5f736a2c3622ebb62bd9d93d8e5d76508b98be922b97160edc3dcca6d8c47022100b23c312ac232a4473f19d2aeb95ab7bdf2b65518911a0d72d50e38b5dd31dc820121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac4730440220508fa761865c8abd81244a168392876ee1d94e8ed83897066b5e2df2400dad24022043f5ee7538e87e9c6aef7ef55133d3e51da7cc522830a9c4d736977a76ef755c0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["OP_CODESEPARATOR in scriptSig is invalid"], [[["10c9f0effe83e97f80f067de2b11c6a00c3088a4bce42c5ae761519af9306f3c", 1, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], - "01000000013c6f30f99a5161e75a2ce4bca488300ca0c6112bde67f0807fe983feeff0c91001000000e608646561646265656675ab61493046022100ce18d384221a731c993939015e3d1bcebafb16e8c0b5b5d14097ec8177ae6f28022100bcab227af90bab33c3fe0a9abfee03ba976ee25dc6ce542526e9b2e56e14b7f10121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac493046022100c3b93edcc0fd6250eb32f2dd8a0bba1754b0f6c3be8ed4100ed582f3db73eba2022100bf75b5bd2eff4d6bf2bda2e34a40fcc07d4aa3cf862ceaa77b47b81eff829f9a01ab21038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000013c6f30f99a5161e75a2ce4bca488300ca0c6112bde67f0807fe983feeff0c91001000000e608646561646265656675ab61493046022100ce18d384221a731c993939015e3d1bcebafb16e8c0b5b5d14097ec8177ae6f28022100bcab227af90bab33c3fe0a9abfee03ba976ee25dc6ce542526e9b2e56e14b7f10121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac493046022100c3b93edcc0fd6250eb32f2dd8a0bba1754b0f6c3be8ed4100ed582f3db73eba2022100bf75b5bd2eff4d6bf2bda2e34a40fcc07d4aa3cf862ceaa77b47b81eff829f9a01ab21038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["Again, FindAndDelete() in scriptSig"], [[["6056ebd549003b10cbbd915cea0d82209fe40b8617104be917a26fa92cbe3d6f", 0, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], - "01000000016f3dbe2ca96fa217e94b1017860be49f20820dea5c91bdcb103b0049d5eb566000000000fd1d0147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac47304402203757e937ba807e4a5da8534c17f9d121176056406a6465054bdd260457515c1a02200f02eccf1bec0f3a0d65df37889143c2e88ab7acec61a7b6f5aa264139141a2b0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000016f3dbe2ca96fa217e94b1017860be49f20820dea5c91bdcb103b0049d5eb566000000000fd1d0147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac47304402203757e937ba807e4a5da8534c17f9d121176056406a6465054bdd260457515c1a02200f02eccf1bec0f3a0d65df37889143c2e88ab7acec61a7b6f5aa264139141a2b0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], [[["5a6b0021a6042a686b6b94abc36b387bef9109847774e8b1e51eb8cc55c53921", 1, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], - "01000000012139c555ccb81ee5b1e87477840991ef7b386bc3ab946b6b682a04a621006b5a01000000fdb40148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f2204148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390175ac4830450220646b72c35beeec51f4d5bc1cbae01863825750d7f490864af354e6ea4f625e9c022100f04b98432df3a9641719dbced53393022e7249fb59db993af1118539830aab870148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a580039017521038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH,CONST_SCRIPTCODE"], + "01000000012139c555ccb81ee5b1e87477840991ef7b386bc3ab946b6b682a04a621006b5a01000000fdb40148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f2204148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390175ac4830450220646b72c35beeec51f4d5bc1cbae01863825750d7f490864af354e6ea4f625e9c022100f04b98432df3a9641719dbced53393022e7249fb59db993af1118539830aab870148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a580039017521038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["FindAndDelete() in redeemScript"], [[["b5b598de91787439afd5938116654e0b16b7a0d0f82742ba37564219c5afcbf9", 0, "DUP HASH160 0x14 0xf6f365c40f0739b61de827a44751e5e99032ed8f EQUALVERIFY CHECKSIG"], @@ -386,7 +387,7 @@ ["FindAndDelete() in bare CHECKMULTISIG"], [[["ceafe58e0f6e7d67c0409fbbf673c84c166e3c5d3c24af58f7175b18df3bb3db", 0, "DUP HASH160 0x14 0xf6f365c40f0739b61de827a44751e5e99032ed8f EQUALVERIFY CHECKSIG"], ["ceafe58e0f6e7d67c0409fbbf673c84c166e3c5d3c24af58f7175b18df3bb3db", 1, "2 0x48 0x3045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 3 CHECKMULTISIG"]], - "0100000002dbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce000000006b4830450221009627444320dc5ef8d7f68f35010b4c050a6ed0d96b67a84db99fda9c9de58b1e02203e4b4aaa019e012e65d69b487fdf8719df72f488fa91506a80c49a33929f1fd50121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffffdbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce010000009300483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303ffffffff01a0860100000000001976a9149bc0bbdd3024da4d0c38ed1aecf5c68dd1d3fa1288ac00000000", "P2SH,CONST_SCRIPTCODE"], + "0100000002dbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce000000006b4830450221009627444320dc5ef8d7f68f35010b4c050a6ed0d96b67a84db99fda9c9de58b1e02203e4b4aaa019e012e65d69b487fdf8719df72f488fa91506a80c49a33929f1fd50121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffffdbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce010000009300483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303ffffffff01a0860100000000001976a9149bc0bbdd3024da4d0c38ed1aecf5c68dd1d3fa1288ac00000000", "CONST_SCRIPTCODE"], ["Make diffs cleaner by leaving a comment here without comma at the end"] ] diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index 11634c90f0..2727af5abd 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -2,7 +2,7 @@ ["The following are deserialized transactions which are valid."], ["They are in the form"], ["[[[prevout hash, prevout index, prevout scriptPubKey, amount?], [input 2], ...],"], -["serializedTransaction, verifyFlags]"], +["serializedTransaction, excluded verifyFlags]"], ["Objects that are only a single string (like this one) are ignored"], ["The following is 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63"], @@ -10,413 +10,413 @@ ["See http://r6.ca/blog/20111119T211504Z.html"], ["It is also the first OP_CHECKMULTISIG transaction in standard form"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000490047304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000490047304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "DERSIG,LOW_S,STRICTENC"], ["The following is a tweaked form of 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63"], ["It is an OP_CHECKMULTISIG with an arbitrary extra byte stuffed into the signature at pos length - 2"], ["The dummy byte is fine however, so the NULLDUMMY flag should be happy"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a0048304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2bab01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH,NULLDUMMY"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a0048304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2bab01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "DERSIG,LOW_S,STRICTENC"], ["The following is a tweaked form of 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63"], ["It is an OP_CHECKMULTISIG with the dummy value set to something other than an empty string"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a01ff47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a01ff47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "DERSIG,LOW_S,STRICTENC,NULLDUMMY"], ["As above, but using an OP_1"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000495147304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000495147304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "DERSIG,LOW_S,STRICTENC,NULLDUMMY"], ["As above, but using an OP_1NEGATE"], [[["60a20bd93aa49ab4b28d514ec10b06e1829ce6818ec06cd3aabd013ebcdc4bb1", 0, "1 0x41 0x04cc71eb30d653c0c3163990c47b976f3fb3f37cccdcbedb169a1dfef58bbfbfaff7d8a473e7e2e6d317b87bafe8bde97e3cf8f065dec022b51d11fcdd0d348ac4 0x41 0x0461cbdcc5409fb4b4d42b51d33381354d80e550078cb532a34bfa2fcfdeb7d76519aecc62770f5b0e4ef8551946d8a540911abe3e7854a26f39f58b25c15342af 2 OP_CHECKMULTISIG"]], -"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000494f47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "P2SH"], +"0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba26000000000494f47304402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e1681a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696ad990364e555c271ad504b88ac00000000", "DERSIG,LOW_S,STRICTENC,NULLDUMMY"], ["The following is c99c49da4c38af669dea436d3e73780dfdb6c1ecf9958baa52960e8baee30e73"], ["It is of interest because it contains a 0-sequence as well as a signature of SIGHASH type 0 (which is not a real type)"], [[["406b2b06bcd34d3c8733e6b79f7a394c8a431fbf4ff5ac705c93f4076bb77602", 0, "DUP HASH160 0x14 0xdc44b1164188067c3a32d4780f5996fa14a4f2d9 EQUALVERIFY CHECKSIG"]], -"01000000010276b76b07f4935c70acf54fbf1f438a4c397a9fb7e633873c4dd3bc062b6b40000000008c493046022100d23459d03ed7e9511a47d13292d3430a04627de6235b6e51a40f9cd386f2abe3022100e7d25b080f0bb8d8d5f878bba7d54ad2fda650ea8d158a33ee3cbd11768191fd004104b0e2c879e4daf7b9ab68350228c159766676a14f5815084ba166432aab46198d4cca98fa3e9981d0a90b2effc514b76279476550ba3663fdcaff94c38420e9d5000000000100093d00000000001976a9149a7b0f3b80c6baaeedce0a0842553800f832ba1f88ac00000000", "P2SH"], +"01000000010276b76b07f4935c70acf54fbf1f438a4c397a9fb7e633873c4dd3bc062b6b40000000008c493046022100d23459d03ed7e9511a47d13292d3430a04627de6235b6e51a40f9cd386f2abe3022100e7d25b080f0bb8d8d5f878bba7d54ad2fda650ea8d158a33ee3cbd11768191fd004104b0e2c879e4daf7b9ab68350228c159766676a14f5815084ba166432aab46198d4cca98fa3e9981d0a90b2effc514b76279476550ba3663fdcaff94c38420e9d5000000000100093d00000000001976a9149a7b0f3b80c6baaeedce0a0842553800f832ba1f88ac00000000", "LOW_S,STRICTENC"], ["A nearly-standard transaction with CHECKSIGVERIFY 1 instead of CHECKSIG"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "NONE"], ["Same as above, but with the signature duplicated in the scriptPubKey with the proper pushdata prefix"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0x5b6462475454710f3c22f5fdf0b40704c92f25c3 EQUALVERIFY CHECKSIGVERIFY 1 0x47 0x3044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a01"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006a473044022067288ea50aa799543a536ff9306f8e1cba05b9c6b10951175b924f96732555ed022026d7b5265f38d21541519e4a1e55044d5b9e17e15cdbaf29ae3792e99e883e7a012103ba8c8b86dea131c22ab967e6dd99bdae8eff7a1f75a2c35f1f944109e3fe5e22ffffffff010000000000000000015100000000", "CLEANSTACK,CONST_SCRIPTCODE"], ["The following is f7fdd091fa6d8f5e7a8c2458f5c38faffff2d3f1406b6e4fe2c99dcc0d2d1cbb"], ["It caught a bug in the workaround for 23b397edccd3740a74adb603c9756370fafcde9bcc4483eb271ecad09a94dd63 in an overly simple implementation. In a signature, it contains an ASN1 integer which isn't strict-DER conformant due to being negative, which doesn't make sense in a signature. Before BIP66 activated, it was a valid signature. After it activated, it's not valid any more."], [[["b464e85df2a238416f8bdae11d120add610380ea07f4ef19c5f9dfd472f96c3d", 0, "DUP HASH160 0x14 0xbef80ecf3a44500fda1bc92176e442891662aed2 EQUALVERIFY CHECKSIG"], ["b7978cc96e59a8b13e0865d3f95657561a7f725be952438637475920bac9eb21", 1, "DUP HASH160 0x14 0xbef80ecf3a44500fda1bc92176e442891662aed2 EQUALVERIFY CHECKSIG"]], -"01000000023d6cf972d4dff9c519eff407ea800361dd0a121de1da8b6f4138a2f25de864b4000000008a4730440220ffda47bfc776bcd269da4832626ac332adfca6dd835e8ecd83cd1ebe7d709b0e022049cffa1cdc102a0b56e0e04913606c70af702a1149dc3b305ab9439288fee090014104266abb36d66eb4218a6dd31f09bb92cf3cfa803c7ea72c1fc80a50f919273e613f895b855fb7465ccbc8919ad1bd4a306c783f22cd3227327694c4fa4c1c439affffffff21ebc9ba20594737864352e95b727f1a565756f9d365083eb1a8596ec98c97b7010000008a4730440220503ff10e9f1e0de731407a4a245531c9ff17676eda461f8ceeb8c06049fa2c810220c008ac34694510298fa60b3f000df01caa244f165b727d4896eb84f81e46bcc4014104266abb36d66eb4218a6dd31f09bb92cf3cfa803c7ea72c1fc80a50f919273e613f895b855fb7465ccbc8919ad1bd4a306c783f22cd3227327694c4fa4c1c439affffffff01f0da5200000000001976a914857ccd42dded6df32949d4646dfa10a92458cfaa88ac00000000", "P2SH"], +"01000000023d6cf972d4dff9c519eff407ea800361dd0a121de1da8b6f4138a2f25de864b4000000008a4730440220ffda47bfc776bcd269da4832626ac332adfca6dd835e8ecd83cd1ebe7d709b0e022049cffa1cdc102a0b56e0e04913606c70af702a1149dc3b305ab9439288fee090014104266abb36d66eb4218a6dd31f09bb92cf3cfa803c7ea72c1fc80a50f919273e613f895b855fb7465ccbc8919ad1bd4a306c783f22cd3227327694c4fa4c1c439affffffff21ebc9ba20594737864352e95b727f1a565756f9d365083eb1a8596ec98c97b7010000008a4730440220503ff10e9f1e0de731407a4a245531c9ff17676eda461f8ceeb8c06049fa2c810220c008ac34694510298fa60b3f000df01caa244f165b727d4896eb84f81e46bcc4014104266abb36d66eb4218a6dd31f09bb92cf3cfa803c7ea72c1fc80a50f919273e613f895b855fb7465ccbc8919ad1bd4a306c783f22cd3227327694c4fa4c1c439affffffff01f0da5200000000001976a914857ccd42dded6df32949d4646dfa10a92458cfaa88ac00000000", "DERSIG,LOW_S,STRICTENC"], ["The following tests for the presence of a bug in the handling of SIGHASH_SINGLE"], ["It results in signing the constant 1, instead of something generated based on the transaction,"], ["when the input doing the signing has an index greater than the maximum output index"], [[["0000000000000000000000000000000000000000000000000000000000000200", 0, "1"], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0xe52b482f2faa8ecbf0db344f93c84ac908557f33 EQUALVERIFY CHECKSIG"]], -"01000000020002000000000000000000000000000000000000000000000000000000000000000000000151ffffffff0001000000000000000000000000000000000000000000000000000000000000000000006b483045022100c9cdd08798a28af9d1baf44a6c77bcc7e279f47dc487c8c899911bc48feaffcc0220503c5c50ae3998a733263c5c0f7061b483e2b56c4c41b456e7d2f5a78a74c077032102d5c25adb51b61339d2b05315791e21bbe80ea470a49db0135720983c905aace0ffffffff010000000000000000015100000000", "P2SH"], +"01000000020002000000000000000000000000000000000000000000000000000000000000000000000151ffffffff0001000000000000000000000000000000000000000000000000000000000000000000006b483045022100c9cdd08798a28af9d1baf44a6c77bcc7e279f47dc487c8c899911bc48feaffcc0220503c5c50ae3998a733263c5c0f7061b483e2b56c4c41b456e7d2f5a78a74c077032102d5c25adb51b61339d2b05315791e21bbe80ea470a49db0135720983c905aace0ffffffff010000000000000000015100000000", "CLEANSTACK"], ["The following tests SIGHASH_SINGLE|SIGHASHANYONECANPAY inputs"], [[["437a1002eb125dec0f93f635763e0ae45f28ff8e81d82945753d0107611cd390", 1, "DUP HASH160 0x14 0x383fb81cb0a3fc724b5e08cf8bbd404336d711f6 EQUALVERIFY CHECKSIG"], ["2d48d32ccad087bcda0ac5b31555bd58d1d2568184cbc8e752dd2be2684af03f", 1, "DUP HASH160 0x14 0x275ec2a233e5b23d43fa19e7bf9beb0cb3996117 EQUALVERIFY CHECKSIG"], ["c76168ef1a272a4f176e55e73157ecfce040cfad16a5272f6296eb7089dca846", 1, "DUP HASH160 0x14 0x34fea2c5a75414fd945273ae2d029ce1f28dafcf EQUALVERIFY CHECKSIG"]], -"010000000390d31c6107013d754529d8818eff285fe40a3e7635f6930fec5d12eb02107a43010000006b483045022100f40815ae3c81a0dd851cc8d376d6fd226c88416671346a9033468cca2cdcc6c202204f764623903e6c4bed1b734b75d82c40f1725e4471a55ad4f51218f86130ac038321033d710ab45bb54ac99618ad23b3c1da661631aa25f23bfe9d22b41876f1d46e4effffffff3ff04a68e22bdd52e7c8cb848156d2d158bd5515b3c50adabc87d0ca2cd3482d010000006a4730440220598d263c107004008e9e26baa1e770be30fd31ee55ded1898f7c00da05a75977022045536bead322ca246779698b9c3df3003377090f41afeca7fb2ce9e328ec4af2832102b738b531def73020bd637f32935924cc88549c8206976226d968edd3a42fc2d7ffffffff46a8dc8970eb96622f27a516adcf40e0fcec5731e7556e174f2a271aef6861c7010000006b483045022100c5b90a777a9fdc90c208dbef7290d1fc1be651f47151ee4ccff646872a454cf90220640cfbc4550446968fbbe9d12528f3adf7d87b31541569c59e790db8a220482583210391332546e22bbe8fe3af54addfad6f8b83d05fa4f5e047593d4c07ae938795beffffffff028036be26000000001976a914ddfb29efad43a667465ac59ff14dc6442a1adfca88ac3d5cba01000000001976a914b64dde7a505a13ca986c40e86e984a8dc81368b688ac00000000", "P2SH"], +"010000000390d31c6107013d754529d8818eff285fe40a3e7635f6930fec5d12eb02107a43010000006b483045022100f40815ae3c81a0dd851cc8d376d6fd226c88416671346a9033468cca2cdcc6c202204f764623903e6c4bed1b734b75d82c40f1725e4471a55ad4f51218f86130ac038321033d710ab45bb54ac99618ad23b3c1da661631aa25f23bfe9d22b41876f1d46e4effffffff3ff04a68e22bdd52e7c8cb848156d2d158bd5515b3c50adabc87d0ca2cd3482d010000006a4730440220598d263c107004008e9e26baa1e770be30fd31ee55ded1898f7c00da05a75977022045536bead322ca246779698b9c3df3003377090f41afeca7fb2ce9e328ec4af2832102b738b531def73020bd637f32935924cc88549c8206976226d968edd3a42fc2d7ffffffff46a8dc8970eb96622f27a516adcf40e0fcec5731e7556e174f2a271aef6861c7010000006b483045022100c5b90a777a9fdc90c208dbef7290d1fc1be651f47151ee4ccff646872a454cf90220640cfbc4550446968fbbe9d12528f3adf7d87b31541569c59e790db8a220482583210391332546e22bbe8fe3af54addfad6f8b83d05fa4f5e047593d4c07ae938795beffffffff028036be26000000001976a914ddfb29efad43a667465ac59ff14dc6442a1adfca88ac3d5cba01000000001976a914b64dde7a505a13ca986c40e86e984a8dc81368b688ac00000000", "NONE"], ["An invalid P2SH Transaction"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000009085768617420697320ffffffff010000000000000000015100000000", "NONE"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000009085768617420697320ffffffff010000000000000000015100000000", "P2SH,CLEANSTACK,WITNESS"], ["A valid P2SH Transaction using the standard transaction type put forth in BIP 16"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x8febbed40483661de6958d957412f82deed8e2f7 EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100c66c9cdf4c43609586d15424c54707156e316d88b0a1534c9e6b0d4f311406310221009c0fe51dbc9c4ab7cc25d3fdbeccf6679fe6827f08edf2b4a9f16ee3eb0e438a0123210338e8034509af564c62644c07691942e0c056752008a173c89f60ab2a88ac2ebfacffffffff010000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100c66c9cdf4c43609586d15424c54707156e316d88b0a1534c9e6b0d4f311406310221009c0fe51dbc9c4ab7cc25d3fdbeccf6679fe6827f08edf2b4a9f16ee3eb0e438a0123210338e8034509af564c62644c07691942e0c056752008a173c89f60ab2a88ac2ebfacffffffff010000000000000000015100000000", "LOW_S"], ["Tests for CheckTransaction()"], ["MAX_MONEY output"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x32afac281462b822adbec5094b8d4d337dd5bd6a EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100e1eadba00d9296c743cb6ecc703fd9ddc9b3cd12906176a226ae4c18d6b00796022100a71aef7d2874deff681ba6080f1b278bac7bb99c61b08a85f4311970ffe7f63f012321030c0588dc44d92bdcbf8e72093466766fdc265ead8db64517b0c542275b70fffbacffffffff010040075af0750700015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006e493046022100e1eadba00d9296c743cb6ecc703fd9ddc9b3cd12906176a226ae4c18d6b00796022100a71aef7d2874deff681ba6080f1b278bac7bb99c61b08a85f4311970ffe7f63f012321030c0588dc44d92bdcbf8e72093466766fdc265ead8db64517b0c542275b70fffbacffffffff010040075af0750700015100000000", "LOW_S"], ["MAX_MONEY output + 0 output"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xb558cbf4930954aa6a344363a15668d7477ae716 EQUAL"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022027deccc14aa6668e78a8c9da3484fbcd4f9dcc9bb7d1b85146314b21b9ae4d86022100d0b43dece8cfb07348de0ca8bc5b86276fa88f7f2138381128b7c36ab2e42264012321029bb13463ddd5d2cc05da6e84e37536cb9525703cfd8f43afdb414988987a92f6acffffffff020040075af075070001510000000000000000015100000000", "P2SH"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000006d483045022027deccc14aa6668e78a8c9da3484fbcd4f9dcc9bb7d1b85146314b21b9ae4d86022100d0b43dece8cfb07348de0ca8bc5b86276fa88f7f2138381128b7c36ab2e42264012321029bb13463ddd5d2cc05da6e84e37536cb9525703cfd8f43afdb414988987a92f6acffffffff020040075af075070001510000000000000000015100000000", "LOW_S"], ["Coinbase of size 2"], ["Note the input is just required to make the tester happy"], [[["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"]], -"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff025151ffffffff010000000000000000015100000000", "P2SH"], +"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff025151ffffffff010000000000000000015100000000", "CLEANSTACK"], ["Coinbase of size 100"], ["Note the input is just required to make the tester happy"], [[["0000000000000000000000000000000000000000000000000000000000000000", -1, "1"]], -"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff6451515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151ffffffff010000000000000000015100000000", "P2SH"], +"01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff6451515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151515151ffffffff010000000000000000015100000000", "CLEANSTACK"], ["Simple transaction with first input is signed with SIGHASH_ALL, second with SIGHASH_ANYONECANPAY"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"], ["0000000000000000000000000000000000000000000000000000000000000200", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"]], - "010000000200010000000000000000000000000000000000000000000000000000000000000000000049483045022100d180fd2eb9140aeb4210c9204d3f358766eb53842b2a9473db687fa24b12a3cc022079781799cd4f038b85135bbe49ec2b57f306b2bb17101b17f71f000fcab2b6fb01ffffffff0002000000000000000000000000000000000000000000000000000000000000000000004847304402205f7530653eea9b38699e476320ab135b74771e1c48b81a5d041e2ca84b9be7a802200ac8d1f40fb026674fe5a5edd3dea715c27baa9baca51ed45ea750ac9dc0a55e81ffffffff010100000000000000015100000000", "P2SH"], + "010000000200010000000000000000000000000000000000000000000000000000000000000000000049483045022100d180fd2eb9140aeb4210c9204d3f358766eb53842b2a9473db687fa24b12a3cc022079781799cd4f038b85135bbe49ec2b57f306b2bb17101b17f71f000fcab2b6fb01ffffffff0002000000000000000000000000000000000000000000000000000000000000000000004847304402205f7530653eea9b38699e476320ab135b74771e1c48b81a5d041e2ca84b9be7a802200ac8d1f40fb026674fe5a5edd3dea715c27baa9baca51ed45ea750ac9dc0a55e81ffffffff010100000000000000015100000000", "NONE"], ["Same as above, but we change the sequence number of the first input to check that SIGHASH_ANYONECANPAY is being followed"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"], ["0000000000000000000000000000000000000000000000000000000000000200", 0, "0x21 0x035e7f0d4d0841bcd56c39337ed086b1a633ee770c1ffdd94ac552a95ac2ce0efc CHECKSIG"]], - "01000000020001000000000000000000000000000000000000000000000000000000000000000000004948304502203a0f5f0e1f2bdbcd04db3061d18f3af70e07f4f467cbc1b8116f267025f5360b022100c792b6e215afc5afc721a351ec413e714305cb749aae3d7fee76621313418df101010000000002000000000000000000000000000000000000000000000000000000000000000000004847304402205f7530653eea9b38699e476320ab135b74771e1c48b81a5d041e2ca84b9be7a802200ac8d1f40fb026674fe5a5edd3dea715c27baa9baca51ed45ea750ac9dc0a55e81ffffffff010100000000000000015100000000", "P2SH"], + "01000000020001000000000000000000000000000000000000000000000000000000000000000000004948304502203a0f5f0e1f2bdbcd04db3061d18f3af70e07f4f467cbc1b8116f267025f5360b022100c792b6e215afc5afc721a351ec413e714305cb749aae3d7fee76621313418df101010000000002000000000000000000000000000000000000000000000000000000000000000000004847304402205f7530653eea9b38699e476320ab135b74771e1c48b81a5d041e2ca84b9be7a802200ac8d1f40fb026674fe5a5edd3dea715c27baa9baca51ed45ea750ac9dc0a55e81ffffffff010100000000000000015100000000", "LOW_S"], ["afd9c17f8913577ec3509520bd6e5d63e9c0fd2a5f70c787993b097ba6ca9fae which has several SIGHASH_SINGLE signatures"], [[["63cfa5a09dc540bf63e53713b82d9ea3692ca97cd608c384f2aa88e51a0aac70", 0, "DUP HASH160 0x14 0xdcf72c4fd02f5a987cf9b02f2fabfcac3341a87d EQUALVERIFY CHECKSIG"], ["04e8d0fcf3846c6734477b98f0f3d4badfb78f020ee097a0be5fe347645b817d", 1, "DUP HASH160 0x14 0xdcf72c4fd02f5a987cf9b02f2fabfcac3341a87d EQUALVERIFY CHECKSIG"], ["ee1377aff5d0579909e11782e1d2f5f7b84d26537be7f5516dd4e43373091f3f", 1, "DUP HASH160 0x14 0xdcf72c4fd02f5a987cf9b02f2fabfcac3341a87d EQUALVERIFY CHECKSIG"]], - "010000000370ac0a1ae588aaf284c308d67ca92c69a39e2db81337e563bf40c59da0a5cf63000000006a4730440220360d20baff382059040ba9be98947fd678fb08aab2bb0c172efa996fd8ece9b702201b4fb0de67f015c90e7ac8a193aeab486a1f587e0f54d0fb9552ef7f5ce6caec032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff7d815b6447e35fbea097e00e028fb7dfbad4f3f0987b4734676c84f3fcd0e804010000006b483045022100c714310be1e3a9ff1c5f7cacc65c2d8e781fc3a88ceb063c6153bf950650802102200b2d0979c76e12bb480da635f192cc8dc6f905380dd4ac1ff35a4f68f462fffd032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff3f1f097333e4d46d51f5e77b53264db8f7f5d2e18217e1099957d0f5af7713ee010000006c493046022100b663499ef73273a3788dea342717c2640ac43c5a1cf862c9e09b206fcb3f6bb8022100b09972e75972d9148f2bdd462e5cb69b57c1214b88fc55ca638676c07cfc10d8032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff0380841e00000000001976a914bfb282c70c4191f45b5a6665cad1682f2c9cfdfb88ac80841e00000000001976a9149857cc07bed33a5cf12b9c5e0500b675d500c81188ace0fd1c00000000001976a91443c52850606c872403c0601e69fa34b26f62db4a88ac00000000", "P2SH"], + "010000000370ac0a1ae588aaf284c308d67ca92c69a39e2db81337e563bf40c59da0a5cf63000000006a4730440220360d20baff382059040ba9be98947fd678fb08aab2bb0c172efa996fd8ece9b702201b4fb0de67f015c90e7ac8a193aeab486a1f587e0f54d0fb9552ef7f5ce6caec032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff7d815b6447e35fbea097e00e028fb7dfbad4f3f0987b4734676c84f3fcd0e804010000006b483045022100c714310be1e3a9ff1c5f7cacc65c2d8e781fc3a88ceb063c6153bf950650802102200b2d0979c76e12bb480da635f192cc8dc6f905380dd4ac1ff35a4f68f462fffd032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff3f1f097333e4d46d51f5e77b53264db8f7f5d2e18217e1099957d0f5af7713ee010000006c493046022100b663499ef73273a3788dea342717c2640ac43c5a1cf862c9e09b206fcb3f6bb8022100b09972e75972d9148f2bdd462e5cb69b57c1214b88fc55ca638676c07cfc10d8032103579ca2e6d107522f012cd00b52b9a65fb46f0c57b9b8b6e377c48f526a44741affffffff0380841e00000000001976a914bfb282c70c4191f45b5a6665cad1682f2c9cfdfb88ac80841e00000000001976a9149857cc07bed33a5cf12b9c5e0500b675d500c81188ace0fd1c00000000001976a91443c52850606c872403c0601e69fa34b26f62db4a88ac00000000", "LOW_S"], ["ddc454a1c0c35c188c98976b17670f69e586d9c0f3593ea879928332f0a069e7, which spends an input that pushes using a PUSHDATA1 that is negative when read as signed"], [[["c5510a5dd97a25f43175af1fe649b707b1df8e1a41489bac33a23087027a2f48", 0, "0x4c 0xae 0x606563686f2022553246736447566b58312b5a536e587574356542793066794778625456415675534a6c376a6a334878416945325364667657734f53474f36633338584d7439435c6e543249584967306a486956304f376e775236644546673d3d22203e20743b206f70656e73736c20656e63202d7061737320706173733a5b314a564d7751432d707269766b65792d6865785d202d64202d6165732d3235362d636263202d61202d696e207460 DROP DUP HASH160 0x14 0xbfd7436b6265aa9de506f8a994f881ff08cc2872 EQUALVERIFY CHECKSIG"]], - "0100000001482f7a028730a233ac9b48411a8edfb107b749e61faf7531f4257ad95d0a51c5000000008b483045022100bf0bbae9bde51ad2b222e87fbf67530fbafc25c903519a1e5dcc52a32ff5844e022028c4d9ad49b006dd59974372a54291d5764be541574bb0c4dc208ec51f80b7190141049dd4aad62741dc27d5f267f7b70682eee22e7e9c1923b9c0957bdae0b96374569b460eb8d5b40d972e8c7c0ad441de3d94c4a29864b212d56050acb980b72b2bffffffff0180969800000000001976a914e336d0017a9d28de99d16472f6ca6d5a3a8ebc9988ac00000000", "P2SH"], + "0100000001482f7a028730a233ac9b48411a8edfb107b749e61faf7531f4257ad95d0a51c5000000008b483045022100bf0bbae9bde51ad2b222e87fbf67530fbafc25c903519a1e5dcc52a32ff5844e022028c4d9ad49b006dd59974372a54291d5764be541574bb0c4dc208ec51f80b7190141049dd4aad62741dc27d5f267f7b70682eee22e7e9c1923b9c0957bdae0b96374569b460eb8d5b40d972e8c7c0ad441de3d94c4a29864b212d56050acb980b72b2bffffffff0180969800000000001976a914e336d0017a9d28de99d16472f6ca6d5a3a8ebc9988ac00000000", "NONE"], ["Correct signature order"], ["Note the input is just required to make the tester happy"], [[["b3da01dd4aae683c7aee4d5d8b52a540a508e1115f77cd7fa9a291243f501223", 0, "HASH160 0x14 0xb1ce99298d5f07364b57b1e5c9cc00be0b04a954 EQUAL"]], -"01000000012312503f2491a2a97fcd775f11e108a540a5528b5d4dee7a3c68ae4add01dab300000000fdfe0000483045022100f6649b0eddfdfd4ad55426663385090d51ee86c3481bdc6b0c18ea6c0ece2c0b0220561c315b07cffa6f7dd9df96dbae9200c2dee09bf93cc35ca05e6cdf613340aa0148304502207aacee820e08b0b174e248abd8d7a34ed63b5da3abedb99934df9fddd65c05c4022100dfe87896ab5ee3df476c2655f9fbe5bd089dccbef3e4ea05b5d121169fe7f5f4014c695221031d11db38972b712a9fe1fc023577c7ae3ddb4a3004187d41c45121eecfdbb5b7210207ec36911b6ad2382860d32989c7b8728e9489d7bbc94a6b5509ef0029be128821024ea9fac06f666a4adc3fc1357b7bec1fd0bdece2b9d08579226a8ebde53058e453aeffffffff0180380100000000001976a914c9b99cddf847d10685a4fabaa0baf505f7c3dfab88ac00000000", "P2SH"], +"01000000012312503f2491a2a97fcd775f11e108a540a5528b5d4dee7a3c68ae4add01dab300000000fdfe0000483045022100f6649b0eddfdfd4ad55426663385090d51ee86c3481bdc6b0c18ea6c0ece2c0b0220561c315b07cffa6f7dd9df96dbae9200c2dee09bf93cc35ca05e6cdf613340aa0148304502207aacee820e08b0b174e248abd8d7a34ed63b5da3abedb99934df9fddd65c05c4022100dfe87896ab5ee3df476c2655f9fbe5bd089dccbef3e4ea05b5d121169fe7f5f4014c695221031d11db38972b712a9fe1fc023577c7ae3ddb4a3004187d41c45121eecfdbb5b7210207ec36911b6ad2382860d32989c7b8728e9489d7bbc94a6b5509ef0029be128821024ea9fac06f666a4adc3fc1357b7bec1fd0bdece2b9d08579226a8ebde53058e453aeffffffff0180380100000000001976a914c9b99cddf847d10685a4fabaa0baf505f7c3dfab88ac00000000", "LOW_S"], ["cc60b1f899ec0a69b7c3f25ddf32c4524096a9c5b01cbd84c6d0312a0c478984, which is a fairly strange transaction which relies on OP_CHECKSIG returning 0 when checking a completely invalid sig of length 0"], [[["cbebc4da731e8995fe97f6fadcd731b36ad40e5ecb31e38e904f6e5982fa09f7", 0, "0x2102085c6600657566acc2d6382a47bc3f324008d2aa10940dd7705a48aa2a5a5e33ac7c2103f5d0fb955f95dd6be6115ce85661db412ec6a08abcbfce7da0ba8297c6cc0ec4ac7c5379a820d68df9e32a147cffa36193c6f7c43a1c8c69cda530e1c6db354bfabdcfefaf3c875379a820f531f3041d3136701ea09067c53e7159c8f9b2746a56c3d82966c54bbc553226879a5479827701200122a59a5379827701200122a59a6353798277537982778779679a68"]], -"0100000001f709fa82596e4f908ee331cb5e0ed46ab331d7dcfaf697fe95891e73dac4ebcb000000008c20ca42095840735e89283fec298e62ac2ddea9b5f34a8cbb7097ad965b87568100201b1b01dc829177da4a14551d2fc96a9db00c6501edfa12f22cd9cefd335c227f483045022100a9df60536df5733dd0de6bc921fab0b3eee6426501b43a228afa2c90072eb5ca02201c78b74266fac7d1db5deff080d8a403743203f109fbcabf6d5a760bf87386d20100ffffffff01c075790000000000232103611f9a45c18f28f06f19076ad571c344c82ce8fcfe34464cf8085217a2d294a6ac00000000", "P2SH"], +"0100000001f709fa82596e4f908ee331cb5e0ed46ab331d7dcfaf697fe95891e73dac4ebcb000000008c20ca42095840735e89283fec298e62ac2ddea9b5f34a8cbb7097ad965b87568100201b1b01dc829177da4a14551d2fc96a9db00c6501edfa12f22cd9cefd335c227f483045022100a9df60536df5733dd0de6bc921fab0b3eee6426501b43a228afa2c90072eb5ca02201c78b74266fac7d1db5deff080d8a403743203f109fbcabf6d5a760bf87386d20100ffffffff01c075790000000000232103611f9a45c18f28f06f19076ad571c344c82ce8fcfe34464cf8085217a2d294a6ac00000000", "CLEANSTACK"], ["Empty pubkey"], [[["229257c295e7f555421c1bfec8538dd30a4b5c37c1c8810bbe83cafa7811652c", 0, "0x00 CHECKSIG NOT"]], -"01000000012c651178faca83be0b81c8c1375c4b0ad38d53c8fe1b1c4255f5e795c25792220000000049483045022100d6044562284ac76c985018fc4a90127847708c9edb280996c507b28babdc4b2a02203d74eca3f1a4d1eea7ff77b528fde6d5dc324ec2dbfdb964ba885f643b9704cd01ffffffff010100000000000000232102c2410f8891ae918cab4ffc4bb4a3b0881be67c7a1e7faa8b5acf9ab8932ec30cac00000000", "P2SH"], +"01000000012c651178faca83be0b81c8c1375c4b0ad38d53c8fe1b1c4255f5e795c25792220000000049483045022100d6044562284ac76c985018fc4a90127847708c9edb280996c507b28babdc4b2a02203d74eca3f1a4d1eea7ff77b528fde6d5dc324ec2dbfdb964ba885f643b9704cd01ffffffff010100000000000000232102c2410f8891ae918cab4ffc4bb4a3b0881be67c7a1e7faa8b5acf9ab8932ec30cac00000000", "STRICTENC,NULLFAIL"], ["Empty signature"], [[["9ca93cfd8e3806b9d9e2ba1cf64e3cc6946ee0119670b1796a09928d14ea25f7", 0, "0x21 0x028a1d66975dbdf97897e3a4aef450ebeb5b5293e4a0b4a6d3a2daaa0b2b110e02 CHECKSIG NOT"]], -"0100000001f725ea148d92096a79b1709611e06e94c63c4ef61cbae2d9b906388efd3ca99c000000000100ffffffff0101000000000000002321028a1d66975dbdf97897e3a4aef450ebeb5b5293e4a0b4a6d3a2daaa0b2b110e02ac00000000", "P2SH"], +"0100000001f725ea148d92096a79b1709611e06e94c63c4ef61cbae2d9b906388efd3ca99c000000000100ffffffff0101000000000000002321028a1d66975dbdf97897e3a4aef450ebeb5b5293e4a0b4a6d3a2daaa0b2b110e02ac00000000", "NONE"], [[["444e00ed7840d41f20ecd9c11d3f91982326c731a02f3c05748414a4fa9e59be", 0, "1 0x00 0x21 0x02136b04758b0b6e363e7a6fbe83aaf527a153db2b060d36cc29f7f8309ba6e458 2 CHECKMULTISIG"]], -"0100000001be599efaa4148474053c2fa031c7262398913f1dc1d9ec201fd44078ed004e44000000004900473044022022b29706cb2ed9ef0cb3c97b72677ca2dfd7b4160f7b4beb3ba806aa856c401502202d1e52582412eba2ed474f1f437a427640306fd3838725fab173ade7fe4eae4a01ffffffff010100000000000000232103ac4bba7e7ca3e873eea49e08132ad30c7f03640b6539e9b59903cf14fd016bbbac00000000", "P2SH"], +"0100000001be599efaa4148474053c2fa031c7262398913f1dc1d9ec201fd44078ed004e44000000004900473044022022b29706cb2ed9ef0cb3c97b72677ca2dfd7b4160f7b4beb3ba806aa856c401502202d1e52582412eba2ed474f1f437a427640306fd3838725fab173ade7fe4eae4a01ffffffff010100000000000000232103ac4bba7e7ca3e873eea49e08132ad30c7f03640b6539e9b59903cf14fd016bbbac00000000", "NONE"], [[["e16abbe80bf30c080f63830c8dbf669deaef08957446e95940227d8c5e6db612", 0, "1 0x21 0x03905380c7013e36e6e19d305311c1b81fce6581f5ee1c86ef0627c68c9362fc9f 0x00 2 CHECKMULTISIG"]], -"010000000112b66d5e8c7d224059e946749508efea9d66bf8d0c83630f080cf30be8bb6ae100000000490047304402206ffe3f14caf38ad5c1544428e99da76ffa5455675ec8d9780fac215ca17953520220779502985e194d84baa36b9bd40a0dbd981163fa191eb884ae83fc5bd1c86b1101ffffffff010100000000000000232103905380c7013e36e6e19d305311c1b81fce6581f5ee1c86ef0627c68c9362fc9fac00000000", "P2SH"], +"010000000112b66d5e8c7d224059e946749508efea9d66bf8d0c83630f080cf30be8bb6ae100000000490047304402206ffe3f14caf38ad5c1544428e99da76ffa5455675ec8d9780fac215ca17953520220779502985e194d84baa36b9bd40a0dbd981163fa191eb884ae83fc5bd1c86b1101ffffffff010100000000000000232103905380c7013e36e6e19d305311c1b81fce6581f5ee1c86ef0627c68c9362fc9fac00000000", "STRICTENC"], [[["ebbcf4bfce13292bd791d6a65a2a858d59adbf737e387e40370d4e64cc70efb0", 0, "2 0x21 0x033bcaa0a602f0d44cc9d5637c6e515b0471db514c020883830b7cefd73af04194 0x21 0x03a88b326f8767f4f192ce252afe33c94d25ab1d24f27f159b3cb3aa691ffe1423 2 CHECKMULTISIG NOT"]], -"0100000001b0ef70cc644e0d37407e387e73bfad598d852a5aa6d691d72b2913cebff4bceb000000004a00473044022068cd4851fc7f9a892ab910df7a24e616f293bcb5c5fbdfbc304a194b26b60fba022078e6da13d8cb881a22939b952c24f88b97afd06b4c47a47d7f804c9a352a6d6d0100ffffffff0101000000000000002321033bcaa0a602f0d44cc9d5637c6e515b0471db514c020883830b7cefd73af04194ac00000000", "P2SH"], +"0100000001b0ef70cc644e0d37407e387e73bfad598d852a5aa6d691d72b2913cebff4bceb000000004a00473044022068cd4851fc7f9a892ab910df7a24e616f293bcb5c5fbdfbc304a194b26b60fba022078e6da13d8cb881a22939b952c24f88b97afd06b4c47a47d7f804c9a352a6d6d0100ffffffff0101000000000000002321033bcaa0a602f0d44cc9d5637c6e515b0471db514c020883830b7cefd73af04194ac00000000", "NULLFAIL"], [[["ba4cd7ae2ad4d4d13ebfc8ab1d93a63e4a6563f25089a18bf0fc68f282aa88c1", 0, "2 0x21 0x037c615d761e71d38903609bf4f46847266edc2fb37532047d747ba47eaae5ffe1 0x21 0x02edc823cd634f2c4033d94f5755207cb6b60c4b1f1f056ad7471c47de5f2e4d50 2 CHECKMULTISIG NOT"]], -"0100000001c188aa82f268fcf08ba18950f263654a3ea6931dabc8bf3ed1d4d42aaed74cba000000004b0000483045022100940378576e069aca261a6b26fb38344e4497ca6751bb10905c76bb689f4222b002204833806b014c26fd801727b792b1260003c55710f87c5adbd7a9cb57446dbc9801ffffffff0101000000000000002321037c615d761e71d38903609bf4f46847266edc2fb37532047d747ba47eaae5ffe1ac00000000", "P2SH"], +"0100000001c188aa82f268fcf08ba18950f263654a3ea6931dabc8bf3ed1d4d42aaed74cba000000004b0000483045022100940378576e069aca261a6b26fb38344e4497ca6751bb10905c76bb689f4222b002204833806b014c26fd801727b792b1260003c55710f87c5adbd7a9cb57446dbc9801ffffffff0101000000000000002321037c615d761e71d38903609bf4f46847266edc2fb37532047d747ba47eaae5ffe1ac00000000", "NULLFAIL"], ["OP_CODESEPARATOR tests"], ["Test that SignatureHash() removes OP_CODESEPARATOR with FindAndDelete()"], [[["bc7fd132fcf817918334822ee6d9bd95c889099c96e07ca2c1eb2cc70db63224", 0, "CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIG"]], -"01000000012432b60dc72cebc1a27ce0969c0989c895bdd9e62e8234839117f8fc32d17fbc000000004a493046022100a576b52051962c25e642c0fd3d77ee6c92487048e5d90818bcf5b51abaccd7900221008204f8fb121be4ec3b24483b1f92d89b1b0548513a134e345c5442e86e8617a501ffffffff010000000000000000016a00000000", "P2SH"], +"01000000012432b60dc72cebc1a27ce0969c0989c895bdd9e62e8234839117f8fc32d17fbc000000004a493046022100a576b52051962c25e642c0fd3d77ee6c92487048e5d90818bcf5b51abaccd7900221008204f8fb121be4ec3b24483b1f92d89b1b0548513a134e345c5442e86e8617a501ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE,LOW_S"], [[["83e194f90b6ef21fa2e3a365b63794fb5daa844bdc9b25de30899fcfe7b01047", 0, "CODESEPARATOR CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIG"]], -"01000000014710b0e7cf9f8930de259bdc4b84aa5dfb9437b665a3e3a21ff26e0bf994e183000000004a493046022100a166121a61b4eeb19d8f922b978ff6ab58ead8a5a5552bf9be73dc9c156873ea02210092ad9bc43ee647da4f6652c320800debcf08ec20a094a0aaf085f63ecb37a17201ffffffff010000000000000000016a00000000", "P2SH"], +"01000000014710b0e7cf9f8930de259bdc4b84aa5dfb9437b665a3e3a21ff26e0bf994e183000000004a493046022100a166121a61b4eeb19d8f922b978ff6ab58ead8a5a5552bf9be73dc9c156873ea02210092ad9bc43ee647da4f6652c320800debcf08ec20a094a0aaf085f63ecb37a17201ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE,LOW_S"], ["Hashed data starts at the CODESEPARATOR"], [[["326882a7f22b5191f1a0cc9962ca4b878cd969cf3b3a70887aece4d801a0ba5e", 0, "0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CODESEPARATOR CHECKSIG"]], -"01000000015ebaa001d8e4ec7a88703a3bcf69d98c874bca6299cca0f191512bf2a7826832000000004948304502203bf754d1c6732fbf87c5dcd81258aefd30f2060d7bd8ac4a5696f7927091dad1022100f5bcb726c4cf5ed0ed34cc13dadeedf628ae1045b7cb34421bc60b89f4cecae701ffffffff010000000000000000016a00000000", "P2SH"], +"01000000015ebaa001d8e4ec7a88703a3bcf69d98c874bca6299cca0f191512bf2a7826832000000004948304502203bf754d1c6732fbf87c5dcd81258aefd30f2060d7bd8ac4a5696f7927091dad1022100f5bcb726c4cf5ed0ed34cc13dadeedf628ae1045b7cb34421bc60b89f4cecae701ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE,LOW_S"], ["But only if execution has reached it"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIGVERIFY CODESEPARATOR 0x21 0x038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041 CHECKSIGVERIFY CODESEPARATOR 1"]], -"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a900000000924830450221009c0a27f886a1d8cb87f6f595fbc3163d28f7a81ec3c4b252ee7f3ac77fd13ffa02203caa8dfa09713c8c4d7ef575c75ed97812072405d932bd11e6a1593a98b679370148304502201e3861ef39a526406bad1e20ecad06be7375ad40ddb582c9be42d26c3a0d7b240221009d0a3985e96522e59635d19cc4448547477396ce0ef17a58e7d74c3ef464292301ffffffff010000000000000000016a00000000", "P2SH"], +"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a900000000924830450221009c0a27f886a1d8cb87f6f595fbc3163d28f7a81ec3c4b252ee7f3ac77fd13ffa02203caa8dfa09713c8c4d7ef575c75ed97812072405d932bd11e6a1593a98b679370148304502201e3861ef39a526406bad1e20ecad06be7375ad40ddb582c9be42d26c3a0d7b240221009d0a3985e96522e59635d19cc4448547477396ce0ef17a58e7d74c3ef464292301ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE,LOW_S"], ["CODESEPARATOR in an unexecuted IF block does not change what is hashed"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], -"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0100ffffffff010000000000000000016a00000000", "P2SH"], +"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a48304502207a6974a77c591fa13dff60cabbb85a0de9e025c09c65a4b2285e47ce8e22f761022100f0efaac9ff8ac36b10721e0aae1fb975c90500b50c56e8a0cc52b0403f0425dd0100ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE,LOW_S"], ["As above, with the IF block executed"], [[["a955032f4d6b0c9bfe8cad8f00a8933790b9c1dc28c82e0f48e75b35da0e4944", 0, "IF CODESEPARATOR ENDIF 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 CHECKSIGVERIFY CODESEPARATOR 1"]], -"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510151ffffffff010000000000000000016a00000000", "P2SH"], +"010000000144490eda355be7480f2ec828dcc1b9903793a8008fad8cfe9b0c6b4d2f0355a9000000004a483045022100fa4a74ba9fd59c59f46c3960cf90cbe0d2b743c471d24a3d5d6db6002af5eebb02204d70ec490fd0f7055a7c45f86514336e3a7f03503dacecabb247fc23f15c83510151ffffffff010000000000000000016a00000000", "CONST_SCRIPTCODE"], ["CHECKSIG is legal in scriptSigs"], [[["ccf7f4053a02e653c36ac75c891b7496d0dc5ce5214f6c913d9cf8f1329ebee0", 0, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], -"0100000001e0be9e32f1f89c3d916c4f21e55cdcd096741b895cc76ac353e6023a05f4f7cc00000000d86149304602210086e5f736a2c3622ebb62bd9d93d8e5d76508b98be922b97160edc3dcca6d8c47022100b23c312ac232a4473f19d2aeb95ab7bdf2b65518911a0d72d50e38b5dd31dc820121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac4730440220508fa761865c8abd81244a168392876ee1d94e8ed83897066b5e2df2400dad24022043f5ee7538e87e9c6aef7ef55133d3e51da7cc522830a9c4d736977a76ef755c0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH"], +"0100000001e0be9e32f1f89c3d916c4f21e55cdcd096741b895cc76ac353e6023a05f4f7cc00000000d86149304602210086e5f736a2c3622ebb62bd9d93d8e5d76508b98be922b97160edc3dcca6d8c47022100b23c312ac232a4473f19d2aeb95ab7bdf2b65518911a0d72d50e38b5dd31dc820121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac4730440220508fa761865c8abd81244a168392876ee1d94e8ed83897066b5e2df2400dad24022043f5ee7538e87e9c6aef7ef55133d3e51da7cc522830a9c4d736977a76ef755c0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "SIGPUSHONLY,CONST_SCRIPTCODE,LOW_S,CLEANSTACK"], ["Same semantics for OP_CODESEPARATOR"], [[["10c9f0effe83e97f80f067de2b11c6a00c3088a4bce42c5ae761519af9306f3c", 1, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], -"01000000013c6f30f99a5161e75a2ce4bca488300ca0c6112bde67f0807fe983feeff0c91001000000e608646561646265656675ab61493046022100ce18d384221a731c993939015e3d1bcebafb16e8c0b5b5d14097ec8177ae6f28022100bcab227af90bab33c3fe0a9abfee03ba976ee25dc6ce542526e9b2e56e14b7f10121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac493046022100c3b93edcc0fd6250eb32f2dd8a0bba1754b0f6c3be8ed4100ed582f3db73eba2022100bf75b5bd2eff4d6bf2bda2e34a40fcc07d4aa3cf862ceaa77b47b81eff829f9a01ab21038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH"], +"01000000013c6f30f99a5161e75a2ce4bca488300ca0c6112bde67f0807fe983feeff0c91001000000e608646561646265656675ab61493046022100ce18d384221a731c993939015e3d1bcebafb16e8c0b5b5d14097ec8177ae6f28022100bcab227af90bab33c3fe0a9abfee03ba976ee25dc6ce542526e9b2e56e14b7f10121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac493046022100c3b93edcc0fd6250eb32f2dd8a0bba1754b0f6c3be8ed4100ed582f3db73eba2022100bf75b5bd2eff4d6bf2bda2e34a40fcc07d4aa3cf862ceaa77b47b81eff829f9a01ab21038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "SIGPUSHONLY,CONST_SCRIPTCODE,LOW_S,CLEANSTACK"], ["Signatures are removed from the script they are in by FindAndDelete() in the CHECKSIG code; even multiple instances of one signature can be removed."], [[["6056ebd549003b10cbbd915cea0d82209fe40b8617104be917a26fa92cbe3d6f", 0, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], -"01000000016f3dbe2ca96fa217e94b1017860be49f20820dea5c91bdcb103b0049d5eb566000000000fd1d0147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac47304402203757e937ba807e4a5da8534c17f9d121176056406a6465054bdd260457515c1a02200f02eccf1bec0f3a0d65df37889143c2e88ab7acec61a7b6f5aa264139141a2b0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH"], +"01000000016f3dbe2ca96fa217e94b1017860be49f20820dea5c91bdcb103b0049d5eb566000000000fd1d0147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140147304402203989ac8f9ad36b5d0919d97fa0a7f70c5272abee3b14477dc646288a8b976df5022027d19da84a066af9053ad3d1d7459d171b7e3a80bc6c4ef7a330677a6be548140121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ac47304402203757e937ba807e4a5da8534c17f9d121176056406a6465054bdd260457515c1a02200f02eccf1bec0f3a0d65df37889143c2e88ab7acec61a7b6f5aa264139141a2b0121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "SIGPUSHONLY,CONST_SCRIPTCODE,CLEANSTACK"], ["That also includes ahead of the opcode being executed."], [[["5a6b0021a6042a686b6b94abc36b387bef9109847774e8b1e51eb8cc55c53921", 1, "DUP HASH160 0x14 0xee5a6aa40facefb2655ac23c0c28c57c65c41f9b EQUALVERIFY CHECKSIG"]], -"01000000012139c555ccb81ee5b1e87477840991ef7b386bc3ab946b6b682a04a621006b5a01000000fdb40148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f2204148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390175ac4830450220646b72c35beeec51f4d5bc1cbae01863825750d7f490864af354e6ea4f625e9c022100f04b98432df3a9641719dbced53393022e7249fb59db993af1118539830aab870148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a580039017521038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "P2SH"], +"01000000012139c555ccb81ee5b1e87477840991ef7b386bc3ab946b6b682a04a621006b5a01000000fdb40148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390121038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f2204148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a5800390175ac4830450220646b72c35beeec51f4d5bc1cbae01863825750d7f490864af354e6ea4f625e9c022100f04b98432df3a9641719dbced53393022e7249fb59db993af1118539830aab870148304502201723e692e5f409a7151db386291b63524c5eb2030df652b1f53022fd8207349f022100b90d9bbf2f3366ce176e5e780a00433da67d9e5c79312c6388312a296a580039017521038479a0fa998cd35259a2ef0a7a5c68662c1474f88ccb6d08a7677bbec7f22041ffffffff010000000000000000016a00000000", "SIGPUSHONLY,CONST_SCRIPTCODE,LOW_S,CLEANSTACK"], ["Finally CHECKMULTISIG removes all signatures prior to hashing the script containing those signatures. In conjunction with the SIGHASH_SINGLE bug this lets us test whether or not FindAndDelete() is actually present in scriptPubKey/redeemScript evaluation by including a signature of the digest 0x01 We can compute in advance for our pubkey, embed it in the scriptPubKey, and then also using a normal SIGHASH_ALL signature. If FindAndDelete() wasn't run, the 'bugged' signature would still be in the hashed script, and the normal signature would fail."], ["Here's an example on mainnet within a P2SH redeemScript. Remarkably it's a standard transaction in <0.9"], [[["b5b598de91787439afd5938116654e0b16b7a0d0f82742ba37564219c5afcbf9", 0, "DUP HASH160 0x14 0xf6f365c40f0739b61de827a44751e5e99032ed8f EQUALVERIFY CHECKSIG"], ["ab9805c6d57d7070d9a42c5176e47bb705023e6b67249fb6760880548298e742", 0, "HASH160 0x14 0xd8dacdadb7462ae15cd906f1878706d0da8660e6 EQUAL"]], -"0100000002f9cbafc519425637ba4227f8d0a0b7160b4e65168193d5af39747891de98b5b5000000006b4830450221008dd619c563e527c47d9bd53534a770b102e40faa87f61433580e04e271ef2f960220029886434e18122b53d5decd25f1f4acb2480659fea20aabd856987ba3c3907e0121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffff42e7988254800876b69f24676b3e0205b77be476512ca4d970707dd5c60598ab00000000fd260100483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a53034930460221008431bdfa72bc67f9d41fe72e94c88fb8f359ffa30b33c72c121c5a877d922e1002210089ef5fc22dd8bfc6bf9ffdb01a9862d27687d424d1fefbab9e9c7176844a187a014c9052483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303210378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71210378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c7153aeffffffff01a08601000000000017a914d8dacdadb7462ae15cd906f1878706d0da8660e68700000000", "P2SH"], +"0100000002f9cbafc519425637ba4227f8d0a0b7160b4e65168193d5af39747891de98b5b5000000006b4830450221008dd619c563e527c47d9bd53534a770b102e40faa87f61433580e04e271ef2f960220029886434e18122b53d5decd25f1f4acb2480659fea20aabd856987ba3c3907e0121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffff42e7988254800876b69f24676b3e0205b77be476512ca4d970707dd5c60598ab00000000fd260100483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a53034930460221008431bdfa72bc67f9d41fe72e94c88fb8f359ffa30b33c72c121c5a877d922e1002210089ef5fc22dd8bfc6bf9ffdb01a9862d27687d424d1fefbab9e9c7176844a187a014c9052483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303210378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71210378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c7153aeffffffff01a08601000000000017a914d8dacdadb7462ae15cd906f1878706d0da8660e68700000000", "CONST_SCRIPTCODE,LOW_S"], ["Same idea, but with bare CHECKMULTISIG"], [[["ceafe58e0f6e7d67c0409fbbf673c84c166e3c5d3c24af58f7175b18df3bb3db", 0, "DUP HASH160 0x14 0xf6f365c40f0739b61de827a44751e5e99032ed8f EQUALVERIFY CHECKSIG"], ["ceafe58e0f6e7d67c0409fbbf673c84c166e3c5d3c24af58f7175b18df3bb3db", 1, "2 0x48 0x3045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 0x21 0x0378d430274f8c5ec1321338151e9f27f4c676a008bdf8638d07c0b6be9ab35c71 3 CHECKMULTISIG"]], -"0100000002dbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce000000006b4830450221009627444320dc5ef8d7f68f35010b4c050a6ed0d96b67a84db99fda9c9de58b1e02203e4b4aaa019e012e65d69b487fdf8719df72f488fa91506a80c49a33929f1fd50121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffffdbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce010000009300483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303ffffffff01a0860100000000001976a9149bc0bbdd3024da4d0c38ed1aecf5c68dd1d3fa1288ac00000000", "P2SH"], +"0100000002dbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce000000006b4830450221009627444320dc5ef8d7f68f35010b4c050a6ed0d96b67a84db99fda9c9de58b1e02203e4b4aaa019e012e65d69b487fdf8719df72f488fa91506a80c49a33929f1fd50121022b78b756e2258af13779c1a1f37ea6800259716ca4b7f0b87610e0bf3ab52a01ffffffffdbb33bdf185b17f758af243c5d3c6e164cc873f6bb9f40c0677d6e0f8ee5afce010000009300483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303483045022015bd0139bcccf990a6af6ec5c1c52ed8222e03a0d51c334df139968525d2fcd20221009f9efe325476eb64c3958e4713e9eefe49bf1d820ed58d2112721b134e2a1a5303ffffffff01a0860100000000001976a9149bc0bbdd3024da4d0c38ed1aecf5c68dd1d3fa1288ac00000000", "CONST_SCRIPTCODE,LOW_S"], ["CHECKLOCKTIMEVERIFY tests"], ["By-height locks, with argument == 0 and == tx nLockTime"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CLEANSTACK"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 CHECKLOCKTIMEVERIFY"]], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "NONE"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ff64cd1d", "CLEANSTACK"], ["By-time locks, with argument just beyond tx nLockTime (but within numerical boundaries)"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "P2SH,CHECKLOCKTIMEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "P2SH,CHECKLOCKTIMEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY"]], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKLOCKTIMEVERIFY"]], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "500000000 CHECKLOCKTIMEVERIFY"]], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000ffffffff", "NONE"], ["Any non-maxint nSequence is fine"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "CLEANSTACK"], ["The argument can be calculated rather than created directly by a PUSHDATA"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 1ADD CHECKLOCKTIMEVERIFY 1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "P2SH,CHECKLOCKTIMEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "499999999 1ADD CHECKLOCKTIMEVERIFY"]], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000065cd1d", "NONE"], ["Perhaps even by an ADD producing a 5-byte result that is out of bounds for other opcodes"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 2147483647 ADD CHECKLOCKTIMEVERIFY 1"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000feffffff", "P2SH,CHECKLOCKTIMEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 2147483647 ADD CHECKLOCKTIMEVERIFY"]], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000feffffff", "NONE"], ["5 byte non-minimally-encoded arguments are valid"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x05 0x0000000000 CHECKLOCKTIMEVERIFY 1"]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CLEANSTACK,MINIMALDATA"], ["Valid CHECKLOCKTIMEVERIFY in scriptSig"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1000000000100000000000000000001000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"01000000010001000000000000000000000000000000000000000000000000000000000000000000000251b1000000000100000000000000000001000000", "SIGPUSHONLY,CLEANSTACK"], ["Valid CHECKLOCKTIMEVERIFY in redeemScript"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xc5b93064159b3b2d6ab506a41b1f50463771b988 EQUAL"]], -"0100000001000100000000000000000000000000000000000000000000000000000000000000000000030251b1000000000100000000000000000001000000", "P2SH,CHECKLOCKTIMEVERIFY"], +"0100000001000100000000000000000000000000000000000000000000000000000000000000000000030251b1000000000100000000000000000001000000", "NONE"], ["A transaction with a non-standard DER signature."], [[["b1dbc81696c8a9c0fccd0693ab66d7c368dbc38c0def4e800685560ddd1b2132", 0, "DUP HASH160 0x14 0x4b3bd7eba3bc0284fd3007be7f3be275e94f5826 EQUALVERIFY CHECKSIG"]], -"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "P2SH"], +"010000000132211bdd0d568506804eef0d8cc3db68c3d766ab9306cdfcc0a9c89616c8dbb1000000006c493045022100c7bb0faea0522e74ff220c20c022d2cb6033f8d167fb89e75a50e237a35fd6d202203064713491b1f8ad5f79e623d0219ad32510bfaa1009ab30cbee77b59317d6e30001210237af13eb2d84e4545af287b919c2282019c9691cc509e78e196a9d8274ed1be0ffffffff0100000000000000001976a914f1b3ed2eda9a2ebe5a9374f692877cdf87c0f95b88ac00000000", "DERSIG,LOW_S,STRICTENC"], ["CHECKSEQUENCEVERIFY tests"], ["By-height locks, with argument == 0 and == txin.nSequence"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "65535 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "65535 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "CLEANSTACK"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "65535 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "65535 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "NONE"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "CLEANSTACK"], ["By-time locks, with argument == 0 and == txin.nSequence"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff40000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff40000100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4259839 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "NONE"], ["Upper sequence with upper sequence is fine"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000800100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000800100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000800100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000800100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000feffffff0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "NONE"], ["Argument 2^31 with various nSequence"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483648 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "NONE"], ["Argument 2^32-1 with various nSequence"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4294967295 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "NONE"], ["Argument 3<<31 with various nSequence"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffbf7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffff7f0100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x050000008001 CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff0100000000000000000000000000", "NONE"], ["5 byte non-minimally-encoded operandss are valid"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x05 0x0000000000 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "MINIMALDATA,CLEANSTACK"], ["The argument can be calculated rather than created directly by a PUSHDATA"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194303 1ADD CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 1SUB CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194303 1ADD CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "4194304 1SUB CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000ffff00000100000000000000000000000000", "NONE"], ["An ADD producing a 5-byte result that sets CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 65536 CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], -[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 4259840 ADD CHECKSEQUENCEVERIFY 1"]], -"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 65536 ADD CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000", "NONE"], +[[["0000000000000000000000000000000000000000000000000000000000000100", 0, "2147483647 4259840 ADD CHECKSEQUENCEVERIFY"]], +"020000000100010000000000000000000000000000000000000000000000000000000000000000000000000040000100000000000000000000000000", "NONE"], ["Valid CHECKSEQUENCEVERIFY in scriptSig"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "1"]], -"02000000010001000000000000000000000000000000000000000000000000000000000000000000000251b2010000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"02000000010001000000000000000000000000000000000000000000000000000000000000000000000251b2010000000100000000000000000000000000", "SIGPUSHONLY,CLEANSTACK"], ["Valid CHECKSEQUENCEVERIFY in redeemScript"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7c17aff532f22beb54069942f9bf567a66133eaf EQUAL"]], -"0200000001000100000000000000000000000000000000000000000000000000000000000000000000030251b2010000000100000000000000000000000000", "P2SH,CHECKSEQUENCEVERIFY"], +"0200000001000100000000000000000000000000000000000000000000000000000000000000000000030251b2010000000100000000000000000000000000", "NONE"], ["Valid P2WPKH (Private key of segwit tests is L5AQtV2HDm4xGsseLokK2VAT2EtYKcTm3c7HwqnJBFt9LdaQULsM)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100cfb07164b36ba64c1b1e8c7720a56ad64d96f6ef332d3d37f9cb3c96477dc44502200a464cd7a9cf94cd70f66ce4f4f0625ef650052c7afcfe29d7d7e01830ff91ed012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100cfb07164b36ba64c1b1e8c7720a56ad64d96f6ef332d3d37f9cb3c96477dc44502200a464cd7a9cf94cd70f66ce4f4f0625ef650052c7afcfe29d7d7e01830ff91ed012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "NONE"], ["Valid P2WSH"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x20 0xff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3db", 1000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100aa5d8aa40a90f23ce2c3d11bc845ca4a12acd99cbea37de6b9f6d86edebba8cb022022dedc2aa0a255f74d04c0b76ece2d7c691f9dd11a64a8ac49f62a99c3a05f9d01232103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ac00000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100aa5d8aa40a90f23ce2c3d11bc845ca4a12acd99cbea37de6b9f6d86edebba8cb022022dedc2aa0a255f74d04c0b76ece2d7c691f9dd11a64a8ac49f62a99c3a05f9d01232103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ac00000000", "NONE"], ["Valid P2SH(P2WPKH)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0xfe9c7dacc9fcfbf7e3b7d5ad06aa2b28c5a7b7e3 EQUAL", 1000]], -"01000000000101000100000000000000000000000000000000000000000000000000000000000000000000171600144c9c3dfac4207d5d8cb89df5722cb3d712385e3fffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100cfb07164b36ba64c1b1e8c7720a56ad64d96f6ef332d3d37f9cb3c96477dc44502200a464cd7a9cf94cd70f66ce4f4f0625ef650052c7afcfe29d7d7e01830ff91ed012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], +"01000000000101000100000000000000000000000000000000000000000000000000000000000000000000171600144c9c3dfac4207d5d8cb89df5722cb3d712385e3fffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100cfb07164b36ba64c1b1e8c7720a56ad64d96f6ef332d3d37f9cb3c96477dc44502200a464cd7a9cf94cd70f66ce4f4f0625ef650052c7afcfe29d7d7e01830ff91ed012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "NONE"], ["Valid P2SH(P2WSH)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x2135ab4f0981830311e35600eebc7376dce3a914 EQUAL", 1000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000023220020ff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100aa5d8aa40a90f23ce2c3d11bc845ca4a12acd99cbea37de6b9f6d86edebba8cb022022dedc2aa0a255f74d04c0b76ece2d7c691f9dd11a64a8ac49f62a99c3a05f9d01232103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ac00000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000023220020ff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbffffffff01e8030000000000001976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac02483045022100aa5d8aa40a90f23ce2c3d11bc845ca4a12acd99cbea37de6b9f6d86edebba8cb022022dedc2aa0a255f74d04c0b76ece2d7c691f9dd11a64a8ac49f62a99c3a05f9d01232103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ac00000000", "NONE"], ["Witness with SigHash Single|AnyoneCanPay"], [[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], -"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff05540b0000000000000151d0070000000000000151840300000000000001513c0f00000000000001512c010000000000000151000248304502210092f4777a0f17bf5aeb8ae768dec5f2c14feabf9d1fe2c89c78dfed0f13fdb86902206da90a86042e252bcd1e80a168c719e4a1ddcc3cebea24b9812c5453c79107e9832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71000000000000", "P2SH,WITNESS"], +"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff05540b0000000000000151d0070000000000000151840300000000000001513c0f00000000000001512c010000000000000151000248304502210092f4777a0f17bf5aeb8ae768dec5f2c14feabf9d1fe2c89c78dfed0f13fdb86902206da90a86042e252bcd1e80a168c719e4a1ddcc3cebea24b9812c5453c79107e9832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71000000000000", "NONE"], ["Witness with SigHash Single|AnyoneCanPay (same signature as previous)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000248304502210092f4777a0f17bf5aeb8ae768dec5f2c14feabf9d1fe2c89c78dfed0f13fdb86902206da90a86042e252bcd1e80a168c719e4a1ddcc3cebea24b9812c5453c79107e9832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000248304502210092f4777a0f17bf5aeb8ae768dec5f2c14feabf9d1fe2c89c78dfed0f13fdb86902206da90a86042e252bcd1e80a168c719e4a1ddcc3cebea24b9812c5453c79107e9832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash Single"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff0484030000000000000151d0070000000000000151540b0000000000000151c800000000000000015100024730440220699e6b0cfe015b64ca3283e6551440a34f901ba62dd4c72fe1cb815afb2e6761022021cc5e84db498b1479de14efda49093219441adc6c543e5534979605e273d80b032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff0484030000000000000151d0070000000000000151540b0000000000000151c800000000000000015100024730440220699e6b0cfe015b64ca3283e6551440a34f901ba62dd4c72fe1cb815afb2e6761022021cc5e84db498b1479de14efda49093219441adc6c543e5534979605e273d80b032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash Single (same signature as previous)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b000000000000015100024730440220699e6b0cfe015b64ca3283e6551440a34f901ba62dd4c72fe1cb815afb2e6761022021cc5e84db498b1479de14efda49093219441adc6c543e5534979605e273d80b032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b000000000000015100024730440220699e6b0cfe015b64ca3283e6551440a34f901ba62dd4c72fe1cb815afb2e6761022021cc5e84db498b1479de14efda49093219441adc6c543e5534979605e273d80b032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash None|AnyoneCanPay"], [[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], -"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff04b60300000000000001519e070000000000000151860b00000000000001009600000000000000015100000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff04b60300000000000001519e070000000000000151860b00000000000001009600000000000000015100000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash None|AnyoneCanPay (same signature as previous)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000248304502210091b32274295c2a3fa02f5bce92fb2789e3fc6ea947fbe1a76e52ea3f4ef2381a022079ad72aefa3837a2e0c033a8652a59731da05fa4a813f4fc48e87c075037256b822103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash None"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff04b60300000000000001519e070000000000000151860b0000000000000100960000000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff04b60300000000000001519e070000000000000151860b0000000000000100960000000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash None (same signature as previous)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash None (same signature, only sequences changed)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"01000000000103000100000000000000000000000000000000000000000000000000000000000000000000000200000000010000000000000000000000000000000000000000000000000000000000000100000000ffffffff000100000000000000000000000000000000000000000000000000000000000002000000000200000003e8030000000000000151d0070000000000000151b80b00000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"01000000000103000100000000000000000000000000000000000000000000000000000000000000000000000200000000010000000000000000000000000000000000000000000000000000000000000100000000ffffffff000100000000000000000000000000000000000000000000000000000000000002000000000200000003e8030000000000000151d0070000000000000151b80b00000000000001510002473044022022fceb54f62f8feea77faac7083c3b56c4676a78f93745adc8a35800bc36adfa022026927df9abcf0a8777829bcfcce3ff0a385fa54c3f9df577405e3ef24ee56479022103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash All|AnyoneCanPay"], [[["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3100], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1100], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 3, "0x51", 4100]], -"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000002483045022100a3cec69b52cba2d2de623eeef89e0ba1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010400010000000000000000000000000000000000000000000000000000000000000200000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000300000000ffffffff03e8030000000000000151d0070000000000000151b80b0000000000000151000002483045022100a3cec69b52cba2d2de623eeef89e0ba1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Witness with SigHash All|AnyoneCanPay (same signature as previous)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623eeef89e0ba1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623eeef89e0ba1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Unknown witness program version (without DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x60 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 2000], ["0000000000000000000000000000000000000000000000000000000000000100", 2, "0x51", 3000]], -"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623ffffffffff1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010300010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000200000000ffffffff03e8030000000000000151d0070000000000000151b80b00000000000001510002483045022100a3cec69b52cba2d2de623ffffffffff1606184ea55476c0f8189fda231bc9cbb022003181ad597f7c380a7d1c740286b1d022b8b04ded028b833282e055e03b8efef812103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], ["Witness with a push of 520 bytes"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x20 0x33198a9bfef674ebddb9ffaa52928017b8472791e54c609cb95f278ac6b1e349", 1000]], -"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015102fd08020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002755100000000", "P2SH,WITNESS"], +"0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff010000000000000000015102fd08020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002755100000000", "NONE"], ["Transaction mixing all SigHash, segwit and normal inputs"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000], @@ -431,67 +431,67 @@ ["0000000000000000000000000000000000000000000000000000000000000100", 9, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1009], ["0000000000000000000000000000000000000000000000000000000000000100", 10, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1010], ["0000000000000000000000000000000000000000000000000000000000000100", 11, "DUP HASH160 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f EQUALVERIFY CHECKSIG", 1011]], -"0100000000010c00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff0001000000000000000000000000000000000000000000000000000000000000020000006a473044022026c2e65b33fcd03b2a3b0f25030f0244bd23cc45ae4dec0f48ae62255b1998a00220463aa3982b718d593a6b9e0044513fd67a5009c2fdccc59992cffc2b167889f4012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000030000006a4730440220008bd8382911218dcb4c9f2e75bf5c5c3635f2f2df49b36994fde85b0be21a1a02205a539ef10fb4c778b522c1be852352ea06c67ab74200977c722b0bc68972575a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000040000006b483045022100d9436c32ff065127d71e1a20e319e4fe0a103ba0272743dbd8580be4659ab5d302203fd62571ee1fe790b182d078ecfd092a509eac112bea558d122974ef9cc012c7012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000050000006a47304402200e2c149b114ec546015c13b2b464bbcb0cdc5872e6775787527af6cbc4830b6c02207e9396c6979fb15a9a2b96ca08a633866eaf20dc0ff3c03e512c1d5a1654f148012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000060000006b483045022100b20e70d897dc15420bccb5e0d3e208d27bdd676af109abbd3f88dbdb7721e6d6022005836e663173fbdfe069f54cde3c2decd3d0ea84378092a5d9d85ec8642e8a41012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff00010000000000000000000000000000000000000000000000000000000000000700000000ffffffff00010000000000000000000000000000000000000000000000000000000000000800000000ffffffff00010000000000000000000000000000000000000000000000000000000000000900000000ffffffff00010000000000000000000000000000000000000000000000000000000000000a00000000ffffffff00010000000000000000000000000000000000000000000000000000000000000b0000006a47304402206639c6e05e3b9d2675a7f3876286bdf7584fe2bbd15e0ce52dd4e02c0092cdc60220757d60b0a61fc95ada79d23746744c72bac1545a75ff6c2c7cdb6ae04e7e9592012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0ce8030000000000000151e9030000000000000151ea030000000000000151eb030000000000000151ec030000000000000151ed030000000000000151ee030000000000000151ef030000000000000151f0030000000000000151f1030000000000000151f2030000000000000151f30300000000000001510248304502210082219a54f61bf126bfc3fa068c6e33831222d1d7138c6faa9d33ca87fd4202d6022063f9902519624254d7c2c8ea7ba2d66ae975e4e229ae38043973ec707d5d4a83012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7102473044022017fb58502475848c1b09f162cb1688d0920ff7f142bed0ef904da2ccc88b168f02201798afa61850c65e77889cbcd648a5703b487895517c88f85cdd18b021ee246a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000000247304402202830b7926e488da75782c81a54cd281720890d1af064629ebf2e31bf9f5435f30220089afaa8b455bbeb7d9b9c3fe1ed37d07685ade8455c76472cda424d93e4074a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7102473044022026326fcdae9207b596c2b05921dbac11d81040c4d40378513670f19d9f4af893022034ecd7a282c0163b89aaa62c22ec202cef4736c58cd251649bad0d8139bcbf55012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71024730440220214978daeb2f38cd426ee6e2f44131a33d6b191af1c216247f1dd7d74c16d84a02205fdc05529b0bc0c430b4d5987264d9d075351c4f4484c16e91662e90a72aab24012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710247304402204a6e9f199dc9672cf2ff8094aaa784363be1eb62b679f7ff2df361124f1dca3302205eeb11f70fab5355c9c8ad1a0700ea355d315e334822fa182227e9815308ee8f012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "P2SH,WITNESS"], +"0100000000010c00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff0001000000000000000000000000000000000000000000000000000000000000020000006a473044022026c2e65b33fcd03b2a3b0f25030f0244bd23cc45ae4dec0f48ae62255b1998a00220463aa3982b718d593a6b9e0044513fd67a5009c2fdccc59992cffc2b167889f4012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000030000006a4730440220008bd8382911218dcb4c9f2e75bf5c5c3635f2f2df49b36994fde85b0be21a1a02205a539ef10fb4c778b522c1be852352ea06c67ab74200977c722b0bc68972575a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000040000006b483045022100d9436c32ff065127d71e1a20e319e4fe0a103ba0272743dbd8580be4659ab5d302203fd62571ee1fe790b182d078ecfd092a509eac112bea558d122974ef9cc012c7012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000050000006a47304402200e2c149b114ec546015c13b2b464bbcb0cdc5872e6775787527af6cbc4830b6c02207e9396c6979fb15a9a2b96ca08a633866eaf20dc0ff3c03e512c1d5a1654f148012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0001000000000000000000000000000000000000000000000000000000000000060000006b483045022100b20e70d897dc15420bccb5e0d3e208d27bdd676af109abbd3f88dbdb7721e6d6022005836e663173fbdfe069f54cde3c2decd3d0ea84378092a5d9d85ec8642e8a41012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff00010000000000000000000000000000000000000000000000000000000000000700000000ffffffff00010000000000000000000000000000000000000000000000000000000000000800000000ffffffff00010000000000000000000000000000000000000000000000000000000000000900000000ffffffff00010000000000000000000000000000000000000000000000000000000000000a00000000ffffffff00010000000000000000000000000000000000000000000000000000000000000b0000006a47304402206639c6e05e3b9d2675a7f3876286bdf7584fe2bbd15e0ce52dd4e02c0092cdc60220757d60b0a61fc95ada79d23746744c72bac1545a75ff6c2c7cdb6ae04e7e9592012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71ffffffff0ce8030000000000000151e9030000000000000151ea030000000000000151eb030000000000000151ec030000000000000151ed030000000000000151ee030000000000000151ef030000000000000151f0030000000000000151f1030000000000000151f2030000000000000151f30300000000000001510248304502210082219a54f61bf126bfc3fa068c6e33831222d1d7138c6faa9d33ca87fd4202d6022063f9902519624254d7c2c8ea7ba2d66ae975e4e229ae38043973ec707d5d4a83012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7102473044022017fb58502475848c1b09f162cb1688d0920ff7f142bed0ef904da2ccc88b168f02201798afa61850c65e77889cbcd648a5703b487895517c88f85cdd18b021ee246a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000000247304402202830b7926e488da75782c81a54cd281720890d1af064629ebf2e31bf9f5435f30220089afaa8b455bbeb7d9b9c3fe1ed37d07685ade8455c76472cda424d93e4074a012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7102473044022026326fcdae9207b596c2b05921dbac11d81040c4d40378513670f19d9f4af893022034ecd7a282c0163b89aaa62c22ec202cef4736c58cd251649bad0d8139bcbf55012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71024730440220214978daeb2f38cd426ee6e2f44131a33d6b191af1c216247f1dd7d74c16d84a02205fdc05529b0bc0c430b4d5987264d9d075351c4f4484c16e91662e90a72aab24012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710247304402204a6e9f199dc9672cf2ff8094aaa784363be1eb62b679f7ff2df361124f1dca3302205eeb11f70fab5355c9c8ad1a0700ea355d315e334822fa182227e9815308ee8f012103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710000000000", "NONE"], ["Unknown version witness program with empty witness"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], ["Witness SIGHASH_SINGLE with output out of bound"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x51", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x20 0x4d6c2a32c87821d68fc016fca70797abdb80df6cd84651d40a9300c6bad79e62", 1000]], -"0100000000010200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff01d00700000000000001510003483045022100e078de4e96a0e05dcdc0a414124dd8475782b5f3f0ed3f607919e9a5eeeb22bf02201de309b3a3109adb3de8074b3610d4cf454c49b61247a2779a0bcbf31c889333032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc711976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac00000000", "P2SH,WITNESS"], +"0100000000010200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff01d00700000000000001510003483045022100e078de4e96a0e05dcdc0a414124dd8475782b5f3f0ed3f607919e9a5eeeb22bf02201de309b3a3109adb3de8074b3610d4cf454c49b61247a2779a0bcbf31c889333032103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc711976a9144c9c3dfac4207d5d8cb89df5722cb3d712385e3f88ac00000000", "NONE"], ["1 byte push should not be considered a witness scriptPubKey"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x01 0x01", 1000]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS,DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "MINIMALDATA,CLEANSTACK"], ["41 bytes push should not be considered a witness scriptPubKey"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x29 0xff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbff0000000000000000", 1000]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS,DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "CLEANSTACK"], ["The witness version must use OP_1 to OP_16 only"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x01 0x10 0x02 0x0001", 1000]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS,DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "MINIMALDATA,CLEANSTACK"], ["The witness program push must be canonical"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x4c02 0x0001", 1000]], -"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS,DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], +"010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "MINIMALDATA,CLEANSTACK"], ["Witness Single|AnyoneCanPay does not hash input's position"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1001]], -"0100000000010200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff02e8030000000000000151e90300000000000001510247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], +"0100000000010200010000000000000000000000000000000000000000000000000000000000000000000000ffffffff00010000000000000000000000000000000000000000000000000000000000000100000000ffffffff02e8030000000000000151e90300000000000001510247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "NONE"], ["Witness Single|AnyoneCanPay does not hash input's position (permutation)"], [[["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1001], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x14 0x4c9c3dfac4207d5d8cb89df5722cb3d712385e3f", 1000]], -"0100000000010200010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff02e9030000000000000151e80300000000000001510248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "P2SH,WITNESS"], +"0100000000010200010000000000000000000000000000000000000000000000000000000000000100000000ffffffff00010000000000000000000000000000000000000000000000000000000000000000000000ffffffff02e9030000000000000151e80300000000000001510248304502210085001a820bfcbc9f9de0298af714493f8a37b3b354bfd21a7097c3e009f2018c022050a8b4dbc8155d4d04da2f5cdd575dcf8dd0108de8bec759bd897ea01ecb3af7832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc710247304402206d59682663faab5e4cb733c562e22cdae59294895929ec38d7c016621ff90da0022063ef0af5f970afe8a45ea836e3509b8847ed39463253106ac17d19c437d3d56b832103596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc7100000000", "NONE"], ["Non witness Single|AnyoneCanPay hash input's position"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x21 0x03596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71 CHECKSIG", 1000], ["0000000000000000000000000000000000000000000000000000000000000100", 1, "0x21 0x03596d3451025c19dbbdeb932d6bf8bfb4ad499b95b6f88db8899efac102e5fc71 CHECKSIG", 1001]], -"01000000020001000000000000000000000000000000000000000000000000000000000000000000004847304402202a0b4b1294d70540235ae033d78e64b4897ec859c7b6f1b2b1d8a02e1d46006702201445e756d2254b0f1dfda9ab8e1e1bc26df9668077403204f32d16a49a36eb6983ffffffff00010000000000000000000000000000000000000000000000000000000000000100000049483045022100acb96cfdbda6dc94b489fd06f2d720983b5f350e31ba906cdbd800773e80b21c02200d74ea5bdf114212b4bbe9ed82c36d2e369e302dff57cb60d01c428f0bd3daab83ffffffff02e8030000000000000151e903000000000000015100000000", "P2SH,WITNESS"], +"01000000020001000000000000000000000000000000000000000000000000000000000000000000004847304402202a0b4b1294d70540235ae033d78e64b4897ec859c7b6f1b2b1d8a02e1d46006702201445e756d2254b0f1dfda9ab8e1e1bc26df9668077403204f32d16a49a36eb6983ffffffff00010000000000000000000000000000000000000000000000000000000000000100000049483045022100acb96cfdbda6dc94b489fd06f2d720983b5f350e31ba906cdbd800773e80b21c02200d74ea5bdf114212b4bbe9ed82c36d2e369e302dff57cb60d01c428f0bd3daab83ffffffff02e8030000000000000151e903000000000000015100000000", "NONE"], ["BIP143 examples: details and private keys are available in BIP143"], ["BIP143 example: P2WSH with OP_CODESEPARATOR and out-of-range SIGHASH_SINGLE."], [[["6eb316926b1c5d567cd6f5e6a84fec606fc53d7b474526d1fff3948020c93dfe", 0, "0x21 0x036d5c20fa14fb2f635474c1dc4ef5909d4568e5569b79fc94d3448486e14685f8 CHECKSIG", 156250000], ["f825690aee1b3dc247da796cacb12687a5e802429fd291cfd63e010f02cf1508", 0, "0x00 0x20 0x5d1b56b63d714eebe542309525f484b7e9d6f686b3781b6f61ef925d66d6f6a0", 4900000000]], -"01000000000102fe3dc9208094f3ffd12645477b3dc56f60ec4fa8e6f5d67c565d1c6b9216b36e000000004847304402200af4e47c9b9629dbecc21f73af989bdaa911f7e6f6c2e9394588a3aa68f81e9902204f3fcf6ade7e5abb1295b6774c8e0abd94ae62217367096bc02ee5e435b67da201ffffffff0815cf020f013ed6cf91d29f4202e8a58726b1ac6c79da47c23d1bee0a6925f80000000000ffffffff0100f2052a010000001976a914a30741f8145e5acadf23f751864167f32e0963f788ac000347304402200de66acf4527789bfda55fc5459e214fa6083f936b430a762c629656216805ac0220396f550692cd347171cbc1ef1f51e15282e837bb2b30860dc77c8f78bc8501e503473044022027dc95ad6b740fe5129e7e62a75dd00f291a2aeb1200b84b09d9e3789406b6c002201a9ecd315dd6a0e632ab20bbb98948bc0c6fb204f2c286963bb48517a7058e27034721026dccc749adc2a9d0d89497ac511f760f45c47dc5ed9cf352a58ac706453880aeadab210255a9626aebf5e29c0e6538428ba0d1dcf6ca98ffdf086aa8ced5e0d0215ea465ac00000000", "P2SH,WITNESS,CONST_SCRIPTCODE"], +"01000000000102fe3dc9208094f3ffd12645477b3dc56f60ec4fa8e6f5d67c565d1c6b9216b36e000000004847304402200af4e47c9b9629dbecc21f73af989bdaa911f7e6f6c2e9394588a3aa68f81e9902204f3fcf6ade7e5abb1295b6774c8e0abd94ae62217367096bc02ee5e435b67da201ffffffff0815cf020f013ed6cf91d29f4202e8a58726b1ac6c79da47c23d1bee0a6925f80000000000ffffffff0100f2052a010000001976a914a30741f8145e5acadf23f751864167f32e0963f788ac000347304402200de66acf4527789bfda55fc5459e214fa6083f936b430a762c629656216805ac0220396f550692cd347171cbc1ef1f51e15282e837bb2b30860dc77c8f78bc8501e503473044022027dc95ad6b740fe5129e7e62a75dd00f291a2aeb1200b84b09d9e3789406b6c002201a9ecd315dd6a0e632ab20bbb98948bc0c6fb204f2c286963bb48517a7058e27034721026dccc749adc2a9d0d89497ac511f760f45c47dc5ed9cf352a58ac706453880aeadab210255a9626aebf5e29c0e6538428ba0d1dcf6ca98ffdf086aa8ced5e0d0215ea465ac00000000", "NONE"], ["BIP143 example: P2WSH with unexecuted OP_CODESEPARATOR and SINGLE|ANYONECANPAY"], [[["01c0cf7fba650638e55eb91261b183251fbb466f90dff17f10086817c542b5e9", 0, "0x00 0x20 0xba468eea561b26301e4cf69fa34bde4ad60c81e70f059f045ca9a79931004a4d", 16777215], ["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215]], -"01000000000102e9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff80e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffff0280969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac80969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS,CONST_SCRIPTCODE"], +"01000000000102e9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff80e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffff0280969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac80969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "NONE"], ["BIP143 example: Same as the previous example with input-output pairs swapped"], [[["1b2a9a426ba603ba357ce7773cb5805cb9c7c2b386d100d1fc9263513188e680", 0, "0x00 0x20 0xd9bbfbe56af7c4b7f960a70d7ea107156913d9e5a26b0a71429df5e097ca6537", 16777215], ["01c0cf7fba650638e55eb91261b183251fbb466f90dff17f10086817c542b5e9", 0, "0x00 0x20 0xba468eea561b26301e4cf69fa34bde4ad60c81e70f059f045ca9a79931004a4d", 16777215]], -"0100000000010280e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffffe9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff0280969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac80969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "P2SH,WITNESS,CONST_SCRIPTCODE"], +"0100000000010280e68831516392fcd100d186b3c2c7b95c80b53c77e77c35ba03a66b429a2a1b0000000000ffffffffe9b542c5176808107ff1df906f46bb1f2583b16112b95ee5380665ba7fcfc0010000000000ffffffff0280969800000000001976a9146648a8cd4531e1ec47f35916de8e259237294d1e88ac80969800000000001976a914de4b231626ef508c9a74a8517e6783c0546d6b2888ac024730440220032521802a76ad7bf74d0e2c218b72cf0cbc867066e2e53db905ba37f130397e02207709e2188ed7f08f4c952d9d13986da504502b8c3be59617e043552f506c46ff83275163ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac02483045022100f6a10b8604e6dc910194b79ccfc93e1bc0ec7c03453caaa8987f7d6c3413566002206216229ede9b4d6ec2d325be245c5b508ff0339bf1794078e20bfe0babc7ffe683270063ab68210392972e2eb617b2388771abe27235fd5ac44af8e61693261550447a4c3e39da98ac00000000", "NONE"], ["BIP143 example: P2SH-P2WSH 6-of-6 multisig signed with 6 different SIGHASH types"], [[["6eb98797a21c6c10aa74edf29d618be109f48a8e94c694f3701e08ca69186436", 1, "HASH160 0x14 0x9993a429037b5d912407a71c252019287b8d27a5 EQUAL", 987654321]], -"0100000000010136641869ca081e70f394c6948e8af409e18b619df2ed74aa106c1ca29787b96e0100000023220020a16b5755f7f6f96dbd65f5f0d6ab9418b89af4b1f14a1bb8a09062c35f0dcb54ffffffff0200e9a435000000001976a914389ffce9cd9ae88dcc0631e88a821ffdbe9bfe2688acc0832f05000000001976a9147480a33f950689af511e6e84c138dbbd3c3ee41588ac080047304402206ac44d672dac41f9b00e28f4df20c52eeb087207e8d758d76d92c6fab3b73e2b0220367750dbbe19290069cba53d096f44530e4f98acaa594810388cf7409a1870ce01473044022068c7946a43232757cbdf9176f009a928e1cd9a1a8c212f15c1e11ac9f2925d9002205b75f937ff2f9f3c1246e547e54f62e027f64eefa2695578cc6432cdabce271502473044022059ebf56d98010a932cf8ecfec54c48e6139ed6adb0728c09cbe1e4fa0915302e022007cd986c8fa870ff5d2b3a89139c9fe7e499259875357e20fcbb15571c76795403483045022100fbefd94bd0a488d50b79102b5dad4ab6ced30c4069f1eaa69a4b5a763414067e02203156c6a5c9cf88f91265f5a942e96213afae16d83321c8b31bb342142a14d16381483045022100a5263ea0553ba89221984bd7f0b13613db16e7a70c549a86de0cc0444141a407022005c360ef0ae5a5d4f9f2f87a56c1546cc8268cab08c73501d6b3be2e1e1a8a08824730440220525406a1482936d5a21888260dc165497a90a15669636d8edca6b9fe490d309c022032af0c646a34a44d1f4576bf6a4a74b67940f8faa84c7df9abe12a01a11e2b4783cf56210307b8ae49ac90a048e9b53357a2354b3334e9c8bee813ecb98e99a7e07e8c3ba32103b28f0c28bfab54554ae8c658ac5c3e0ce6e79ad336331f78c428dd43eea8449b21034b8113d703413d57761b8b9781957b8c0ac1dfe69f492580ca4195f50376ba4a21033400f6afecb833092a9a21cfdf1ed1376e58c5d1f47de74683123987e967a8f42103a6d48b1131e94ba04d9737d61acdaa1322008af9602b3b14862c07a1789aac162102d8b661b0b3302ee2f162b09e07a55ad5dfbe673a9f01d9f0c19617681024306b56ae00000000", "P2SH,WITNESS"], +"0100000000010136641869ca081e70f394c6948e8af409e18b619df2ed74aa106c1ca29787b96e0100000023220020a16b5755f7f6f96dbd65f5f0d6ab9418b89af4b1f14a1bb8a09062c35f0dcb54ffffffff0200e9a435000000001976a914389ffce9cd9ae88dcc0631e88a821ffdbe9bfe2688acc0832f05000000001976a9147480a33f950689af511e6e84c138dbbd3c3ee41588ac080047304402206ac44d672dac41f9b00e28f4df20c52eeb087207e8d758d76d92c6fab3b73e2b0220367750dbbe19290069cba53d096f44530e4f98acaa594810388cf7409a1870ce01473044022068c7946a43232757cbdf9176f009a928e1cd9a1a8c212f15c1e11ac9f2925d9002205b75f937ff2f9f3c1246e547e54f62e027f64eefa2695578cc6432cdabce271502473044022059ebf56d98010a932cf8ecfec54c48e6139ed6adb0728c09cbe1e4fa0915302e022007cd986c8fa870ff5d2b3a89139c9fe7e499259875357e20fcbb15571c76795403483045022100fbefd94bd0a488d50b79102b5dad4ab6ced30c4069f1eaa69a4b5a763414067e02203156c6a5c9cf88f91265f5a942e96213afae16d83321c8b31bb342142a14d16381483045022100a5263ea0553ba89221984bd7f0b13613db16e7a70c549a86de0cc0444141a407022005c360ef0ae5a5d4f9f2f87a56c1546cc8268cab08c73501d6b3be2e1e1a8a08824730440220525406a1482936d5a21888260dc165497a90a15669636d8edca6b9fe490d309c022032af0c646a34a44d1f4576bf6a4a74b67940f8faa84c7df9abe12a01a11e2b4783cf56210307b8ae49ac90a048e9b53357a2354b3334e9c8bee813ecb98e99a7e07e8c3ba32103b28f0c28bfab54554ae8c658ac5c3e0ce6e79ad336331f78c428dd43eea8449b21034b8113d703413d57761b8b9781957b8c0ac1dfe69f492580ca4195f50376ba4a21033400f6afecb833092a9a21cfdf1ed1376e58c5d1f47de74683123987e967a8f42103a6d48b1131e94ba04d9737d61acdaa1322008af9602b3b14862c07a1789aac162102d8b661b0b3302ee2f162b09e07a55ad5dfbe673a9f01d9f0c19617681024306b56ae00000000", "NONE"], ["FindAndDelete tests"], ["This is a test of FindAndDelete. The first tx is a spend of normal P2SH and the second tx is a spend of bare P2WSH."], @@ -501,24 +501,24 @@ ["This is to show that FindAndDelete is applied only to non-segwit scripts"], ["Non-segwit: correct sighash (with FindAndDelete) = 1ba1fe3bc90c5d1265460e684ce6774e324f0fabdf67619eda729e64e8b6bc08"], [[["f18783ace138abac5d3a7a5cf08e88fe6912f267ef936452e0c27d090621c169", 7000, "HASH160 0x14 0x0c746489e2d83cdbb5b90b432773342ba809c134 EQUAL", 200000]], -"010000000169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f1581b0000b64830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0121037a3fb04bcdb09eba90f69961ba1692a3528e45e67c85b200df820212d7594d334aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e01ffffffff0101000000000000000000000000", "P2SH,WITNESS"], +"010000000169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f1581b0000b64830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0121037a3fb04bcdb09eba90f69961ba1692a3528e45e67c85b200df820212d7594d334aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e01ffffffff0101000000000000000000000000", "CONST_SCRIPTCODE,LOW_S"], ["BIP143: correct sighash (without FindAndDelete) = 71c9cd9b2869b9c70b01b1f0360c148f42dee72297db312638df136f43311f23"], [[["f18783ace138abac5d3a7a5cf08e88fe6912f267ef936452e0c27d090621c169", 7500, "0x00 0x20 0x9e1be07558ea5cc8e02ed1d80c0911048afad949affa36d5c3951e3159dbea19", 200000]], -"0100000000010169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f14c1d000000ffffffff01010000000000000000034830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e012102a9781d66b61fb5a7ef00ac5ad5bc6ffc78be7b44a566e3c87870e1079368df4c4aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0100000000", "P2SH,WITNESS,CONST_SCRIPTCODE"], +"0100000000010169c12106097dc2e0526493ef67f21269fe888ef05c7a3a5dacab38e1ac8387f14c1d000000ffffffff01010000000000000000034830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e012102a9781d66b61fb5a7ef00ac5ad5bc6ffc78be7b44a566e3c87870e1079368df4c4aad4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0100000000", "LOW_S"], ["This is multisig version of the FindAndDelete tests"], ["Script is 2 CHECKMULTISIGVERIFY <sig1> <sig2> DROP"], ["52af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960175"], ["Signature is 0 <sig1> <sig2> 2 <key1> <key2>"], ["Non-segwit: correct sighash (with FindAndDelete) = 1d50f00ba4db2917b903b0ec5002e017343bb38876398c9510570f5dce099295"], [[["9628667ad48219a169b41b020800162287d2c0f713c04157e95c484a8dcb7592", 7000, "HASH160 0x14 0x5748407f5ca5cdca53ba30b79040260770c9ee1b EQUAL", 200000]], -"01000000019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a662896581b0000fd6f01004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c03959601522102cd74a2809ffeeed0092bc124fd79836706e41f048db3f6ae9df8708cefb83a1c2102e615999372426e46fd107b76eaf007156a507584aa2cc21de9eee3bdbd26d36c4c9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960175ffffffff0101000000000000000000000000", "P2SH,WITNESS"], +"01000000019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a662896581b0000fd6f01004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c03959601522102cd74a2809ffeeed0092bc124fd79836706e41f048db3f6ae9df8708cefb83a1c2102e615999372426e46fd107b76eaf007156a507584aa2cc21de9eee3bdbd26d36c4c9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960175ffffffff0101000000000000000000000000", "CONST_SCRIPTCODE,LOW_S"], ["BIP143: correct sighash (without FindAndDelete) = c1628a1e7c67f14ca0c27c06e4fdeec2e6d1a73c7a91d7c046ff83e835aebb72"], [[["9628667ad48219a169b41b020800162287d2c0f713c04157e95c484a8dcb7592", 7500, "0x00 0x20 0x9b66c15b4e0b4eb49fa877982cafded24859fe5b0e2dbfbe4f0df1de7743fd52", 200000]], -"010000000001019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a6628964c1d000000ffffffff0101000000000000000007004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960101022102966f109c54e85d3aee8321301136cedeb9fc710fdef58a9de8a73942f8e567c021034ffc99dd9a79dd3cb31e2ab3e0b09e0e67db41ac068c625cd1f491576016c84e9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c039596017500000000", "P2SH,WITNESS,CONST_SCRIPTCODE"], +"010000000001019275cb8d4a485ce95741c013f7c0d28722160008021bb469a11982d47a6628964c1d000000ffffffff0101000000000000000007004830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c0395960101022102966f109c54e85d3aee8321301136cedeb9fc710fdef58a9de8a73942f8e567c021034ffc99dd9a79dd3cb31e2ab3e0b09e0e67db41ac068c625cd1f491576016c84e9552af4830450220487fb382c4974de3f7d834c1b617fe15860828c7f96454490edd6d891556dcc9022100baf95feb48f845d5bfc9882eb6aeefa1bc3790e39f59eaa46ff7f15ae626c53e0148304502205286f726690b2e9b0207f0345711e63fa7012045b9eb0f19c2458ce1db90cf43022100e89f17f86abc5b149eba4115d4f128bcf45d77fb3ecdd34f594091340c039596017500000000", "LOW_S"], ["Test long outputs, which are streamed using length-prefixed bitcoin strings. This might be surprising."], [[["1111111111111111111111111111111111111111111111111111111111111111", 0, "0x00 0x14 0x751e76e8199196d454941c45d1b3a323f1433bd6", 5000000]], -"0100000000010111111111111111111111111111111111111111111111111111111111111111110000000000ffffffff0130244c0000000000fd02014cdc1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111175210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ac02483045022100c1a4a6581996a7fdfea77d58d537955a5655c1d619b6f3ab6874f28bb2e19708022056402db6fede03caae045a3be616a1a2d0919a475ed4be828dc9ff21f24063aa01210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179800000000", "P2SH,WITNESS"], +"0100000000010111111111111111111111111111111111111111111111111111111111111111110000000000ffffffff0130244c0000000000fd02014cdc1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111175210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ac02483045022100c1a4a6581996a7fdfea77d58d537955a5655c1d619b6f3ab6874f28bb2e19708022056402db6fede03caae045a3be616a1a2d0919a475ed4be828dc9ff21f24063aa01210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179800000000", "NONE"], ["Make diffs cleaner by leaving a comment here without comma at the end"] ] diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index d926f8d767..0d480e35ea 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -80,12 +80,12 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = std::make_unique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(chainparams, *connman, nullptr, *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr1, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false); dummyNode1.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerManager &peerLogic, CConnmanTest* connman) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); - vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY)); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false)); CNode &node = *vNodes.back(); node.SetCommonVersion(PROTOCOL_VERSION); @@ -150,8 +150,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = std::make_unique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(chainparams, *connman, nullptr, *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -224,12 +224,12 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = std::make_unique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(chainparams, *connman, banman.get(), *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode1(id++, NODE_NETWORK, INVALID_SOCKET, addr1, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); dummyNode1.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); dummyNode1.fSuccessfullyConnected = true; @@ -242,7 +242,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode2(id++, NODE_NETWORK, INVALID_SOCKET, addr2, /* nKeyedNetGroupIn */ 1, /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); dummyNode2.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode2); dummyNode2.fSuccessfullyConnected = true; @@ -271,15 +271,15 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = std::make_unique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(chainparams, *connman, banman.get(), *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); banman->ClearBanned(); int64_t nStartTime = GetTime(); SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, INVALID_SOCKET, addr, 4, 4, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode(id++, NODE_NETWORK, INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 4, /* nLocalHostNonceIn */ 4, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); dummyNode.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode); dummyNode.fSuccessfullyConnected = true; diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 20132d5782..acbd6a01ee 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -65,7 +65,7 @@ std::string UseHInsteadOfApostrophe(const std::string& desc) const std::set<std::vector<uint32_t>> ONLY_EMPTY{{}}; -void DoCheck(const std::string& prv, const std::string& pub, int flags, const std::vector<std::vector<std::string>>& scripts, const Optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, +void DoCheck(const std::string& prv, const std::string& pub, const std::string& norm_prv, const std::string& norm_pub, int flags, const std::vector<std::vector<std::string>>& scripts, const Optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, bool replace_apostrophe_with_h_in_prv=false, bool replace_apostrophe_with_h_in_pub=false) { FlatSigningProvider keys_priv, keys_pub; @@ -112,6 +112,17 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st BOOST_CHECK(EqualDescriptor(prv, prv1)); BOOST_CHECK(!parse_pub->ToPrivateString(keys_pub, prv1)); + // Check that private can produce the normalized descriptors + std::string norm1; + BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1, false)); + BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1, false)); + BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1, true)); + BOOST_CHECK(EqualDescriptor(norm1, norm_prv)); + BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1, true)); + BOOST_CHECK(EqualDescriptor(norm1, norm_prv)); + // Check whether IsRange on both returns the expected result BOOST_CHECK_EQUAL(parse_pub->IsRange(), (flags & RANGE) != 0); BOOST_CHECK_EQUAL(parse_priv->IsRange(), (flags & RANGE) != 0); @@ -251,29 +262,29 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st BOOST_CHECK_MESSAGE(left_paths.empty(), "Not all expected key paths found: " + prv); } -void Check(const std::string& prv, const std::string& pub, int flags, const std::vector<std::vector<std::string>>& scripts, const Optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY) +void Check(const std::string& prv, const std::string& pub, const std::string& norm_prv, const std::string& norm_pub, int flags, const std::vector<std::vector<std::string>>& scripts, const Optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY) { bool found_apostrophes_in_prv = false; bool found_apostrophes_in_pub = false; // Do not replace apostrophes with 'h' in prv and pub - DoCheck(prv, pub, flags, scripts, type, paths); + DoCheck(prv, pub, norm_prv, norm_pub, flags, scripts, type, paths); // Replace apostrophes with 'h' in prv but not in pub, if apostrophes are found in prv if (prv.find('\'') != std::string::npos) { found_apostrophes_in_prv = true; - DoCheck(prv, pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */false); + DoCheck(prv, pub, norm_prv, norm_pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */false); } // Replace apostrophes with 'h' in pub but not in prv, if apostrophes are found in pub if (pub.find('\'') != std::string::npos) { found_apostrophes_in_pub = true; - DoCheck(prv, pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */false, /*replace_apostrophe_with_h_in_pub = */true); + DoCheck(prv, pub, norm_prv, norm_pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */false, /*replace_apostrophe_with_h_in_pub = */true); } // Replace apostrophes with 'h' both in prv and in pub, if apostrophes are found in both if (found_apostrophes_in_prv && found_apostrophes_in_pub) { - DoCheck(prv, pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */true); + DoCheck(prv, pub, norm_prv, norm_pub, flags, scripts, type, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */true); } } @@ -284,50 +295,51 @@ BOOST_FIXTURE_TEST_SUITE(descriptor_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(descriptor_test) { // Basic single-key compressed - Check("combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac","76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac","00149a1c78a507689f6f54b847ad1cef1e614ee23f1e","a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, nullopt); - Check("pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"}}, nullopt); - Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); - Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); - Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); + Check("combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac","76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac","00149a1c78a507689f6f54b847ad1cef1e614ee23f1e","a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, nullopt); + Check("pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"}}, nullopt); + Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); + Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); + Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin // Basic single-key uncompressed - Check("combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac","76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, nullopt); - Check("pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac"}}, nullopt); - Check("pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, OutputType::LEGACY); + Check("combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "combo(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "combo(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)",SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac","76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, nullopt); + Check("pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235ac"}}, nullopt); + Check("pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "pkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "pkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"76a914b5bd079c4d57cc7fc28ecf8213a6b791625b818388ac"}}, OutputType::LEGACY); CheckUnparsable("wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Uncompressed keys are not allowed"); // No uncompressed keys in witness CheckUnparsable("wsh(pk(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "wsh(pk(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness CheckUnparsable("sh(wpkh(5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss))", "sh(wpkh(04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235))", "Uncompressed keys are not allowed"); // No uncompressed keys in witness // Some unconventional single-key constructions - Check("sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141857af51a5e516552b3086430fd8ce55f7c1a52487"}}, OutputType::LEGACY); - Check("sh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141a31ad23bf49c247dd531a623c2ef57da3c400c587"}}, OutputType::LEGACY); - Check("wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"00202e271faa2325c199d25d22e1ead982e45b64eeb4f31e73dbdf41bd4b5fec23fa"}}, OutputType::BECH32); - Check("wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"0020338e023079b91c58571b20e602d7805fb808c22473cbc391a41b1bd3a192e75b"}}, OutputType::BECH32); - Check("sh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", SIGNABLE, {{"a91472d0c5a3bfad8c3e7bd5303a72b94240e80b6f1787"}}, OutputType::P2SH_SEGWIT); - Check("sh(wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", SIGNABLE, {{"a914b61b92e2ca21bac1e72a3ab859a742982bea960a87"}}, OutputType::P2SH_SEGWIT); + Check("sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141857af51a5e516552b3086430fd8ce55f7c1a52487"}}, OutputType::LEGACY); + Check("sh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a9141a31ad23bf49c247dd531a623c2ef57da3c400c587"}}, OutputType::LEGACY); + Check("wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"00202e271faa2325c199d25d22e1ead982e45b64eeb4f31e73dbdf41bd4b5fec23fa"}}, OutputType::BECH32); + Check("wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"0020338e023079b91c58571b20e602d7805fb808c22473cbc391a41b1bd3a192e75b"}}, OutputType::BECH32); + Check("sh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", "sh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", SIGNABLE, {{"a91472d0c5a3bfad8c3e7bd5303a72b94240e80b6f1787"}}, OutputType::P2SH_SEGWIT); + Check("sh(wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", "sh(wsh(pkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(wsh(pkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", SIGNABLE, {{"a914b61b92e2ca21bac1e72a3ab859a742982bea960a87"}}, OutputType::P2SH_SEGWIT); // Versions with BIP32 derivations - Check("combo([01234567]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", SIGNABLE, {{"2102d2b36900396c9282fa14628566582f206a5dd0bcc8d5e892611806cafb0301f0ac","76a91431a507b815593dfc51ffc7245ae7e5aee304246e88ac","001431a507b815593dfc51ffc7245ae7e5aee304246e","a9142aafb926eb247cb18240a7f4c07983ad1f37922687"}}, nullopt); - Check("pk(xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", DEFAULT, {{"210379e45b3cf75f9c5f9befd8e9506fb962f6a9d185ac87001ec44a8d3df8d4a9e3ac"}}, nullopt, {{0}}); - Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); - Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); - Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); - Check("combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", RANGE, {{"2102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e076ac","76a914f90e3178ca25f2c808dc76624032d352fdbdfaf288ac","0014f90e3178ca25f2c808dc76624032d352fdbdfaf2","a91408f3ea8c68d4a7585bf9e8bda226723f70e445f087"},{"21032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ecac","76a914a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b788ac","0014a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b7","a91473e39884cb71ae4e5ac9739e9225026c99763e6687"}}, nullopt, {{0}, {1}}); + Check("combo([01234567]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo([01234567]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", SIGNABLE, {{"2102d2b36900396c9282fa14628566582f206a5dd0bcc8d5e892611806cafb0301f0ac","76a91431a507b815593dfc51ffc7245ae7e5aee304246e88ac","001431a507b815593dfc51ffc7245ae7e5aee304246e","a9142aafb926eb247cb18240a7f4c07983ad1f37922687"}}, nullopt); + Check("pk(xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", "pk(xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", DEFAULT, {{"210379e45b3cf75f9c5f9befd8e9506fb962f6a9d185ac87001ec44a8d3df8d4a9e3ac"}}, nullopt, {{0}}); + Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([bd16bee5/2147483647']xprv9vHkqa6XAPwKqSKSEJMcAB3yoCZhaSVsGZbSkFY5L3Lfjjk8sjZucbsbvEw5o3QrSA69nPfZDCgFnNnLhQ2ohpZuwummndnPasDw2Qr6dC2/0)", "pkh([bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); + Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); + Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", "combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", RANGE, {{"2102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e076ac","76a914f90e3178ca25f2c808dc76624032d352fdbdfaf288ac","0014f90e3178ca25f2c808dc76624032d352fdbdfaf2","a91408f3ea8c68d4a7585bf9e8bda226723f70e445f087"},{"21032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ecac","76a914a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b788ac","0014a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b7","a91473e39884cb71ae4e5ac9739e9225026c99763e6687"}}, nullopt, {{0}, {1}}); CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "Key path value 2147483648 is out of range"); // BIP 32 path element overflow CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "Key path value '1aa' is not a valid uint32"); // Path is not valid uint + Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647']xprv9vHkqa6XAPwKqSKSEJMcAB3yoCZhaSVsGZbSkFY5L3Lfjjk8sjZucbsbvEw5o3QrSA69nPfZDCgFnNnLhQ2ohpZuwummndnPasDw2Qr6dC2/0)", "pkh([01234567/10/20/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); // Multisig constructions - Check("multi(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); - Check("sortedmulti(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); - Check("sortedmulti(1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); - Check("sortedmulti(2,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/*,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", RANGE, {{"5221025d5fc65ebb8d44a5274b53bac21ff8307fec2334a32df05553459f8b1f7fe1b62102fbd47cc8034098f0e6a94c6aeee8528abf0a2153a5d8e46d325b7284c046784652ae"}, {"52210264fd4d1f5dea8ded94c61e9641309349b62f27fbffe807291f664e286bfbe6472103f4ece6dfccfa37b211eb3d0af4d0c61dba9ef698622dc17eecdf764beeb005a652ae"}, {"5221022ccabda84c30bad578b13c89eb3b9544ce149787e5b538175b1d1ba259cbb83321024d902e1a2fc7a8755ab5b694c575fce742c48d9ff192e63df5193e4c7afe1f9c52ae"}}, nullopt, {{0}, {1}, {2}, {0, 0, 0}, {0, 0, 1}, {0, 0, 2}}); - Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); - Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); + Check("multi(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "multi(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); + Check("sortedmulti(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "sortedmulti(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); + Check("sortedmulti(1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "sortedmulti(1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, nullopt); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sortedmulti(2,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/*,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", "sortedmulti(2,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/*,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", RANGE, {{"5221025d5fc65ebb8d44a5274b53bac21ff8307fec2334a32df05553459f8b1f7fe1b62102fbd47cc8034098f0e6a94c6aeee8528abf0a2153a5d8e46d325b7284c046784652ae"}, {"52210264fd4d1f5dea8ded94c61e9641309349b62f27fbffe807291f664e286bfbe6472103f4ece6dfccfa37b211eb3d0af4d0c61dba9ef698622dc17eecdf764beeb005a652ae"}, {"5221022ccabda84c30bad578b13c89eb3b9544ce149787e5b538175b1d1ba259cbb83321024d902e1a2fc7a8755ab5b694c575fce742c48d9ff192e63df5193e4c7afe1f9c52ae"}}, nullopt, {{0}, {1}, {2}, {0, 0, 0}, {0, 0, 1}, {0, 0, 2}}); + Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647']xprv9vHkqa6XAPwKqSKSEJMcAB3yoCZhaSVsGZbSkFY5L3Lfjjk8sjZucbsbvEw5o3QrSA69nPfZDCgFnNnLhQ2ohpZuwummndnPasDw2Qr6dC2/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", "sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232))", "P2SH script is too large, 547 bytes is larger than 520 bytes"); // P2SH does not fit 16 compressed pubkeys in a redeemscript CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multiple ']' characters found for a single pubkey"); // Double key origin descriptor CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint @@ -350,8 +362,8 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("wsh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", "Cannot have wsh within wsh"); // Cannot embed P2WSH inside P2WSH // Checksums - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", "sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#", "Expected 8 character checksum, not 0 characters"); // Empty checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfyq", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5tq", "Expected 8 character checksum, not 9 characters"); // Too long checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxf", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5", "Expected 8 character checksum, not 7 characters"); // Too short checksum diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index d02c3613ba..e52cd5230c 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <test/util/setup_common.h> #include <util/system.h> +#include <util/getuniquepath.h> #include <boost/test/unit_test.hpp> @@ -52,6 +53,38 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) file >> input_buffer; BOOST_CHECK_EQUAL(input_buffer, "bitcoin"); } + { + // Join an absolute path and a relative path. + fs::path p = fsbridge::AbsPathJoin(tmpfolder, "fs_tests_₿_🏃"); + BOOST_CHECK(p.is_absolute()); + BOOST_CHECK_EQUAL(tmpfile1, p); + } + { + // Join two absolute paths. + fs::path p = fsbridge::AbsPathJoin(tmpfile1, tmpfile2); + BOOST_CHECK(p.is_absolute()); + BOOST_CHECK_EQUAL(tmpfile2, p); + } + { + // Ensure joining with empty paths does not add trailing path components. + BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, "")); + BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, {})); + } + { + fs::path p1 = GetUniquePath(tmpfolder); + fs::path p2 = GetUniquePath(tmpfolder); + fs::path p3 = GetUniquePath(tmpfolder); + + // Ensure that the parent path is always the same. + BOOST_CHECK_EQUAL(tmpfolder, p1.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p2.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p3.parent_path()); + + // Ensure that generated paths are actually different. + BOOST_CHECK(p1 != p2); + BOOST_CHECK(p2 != p3); + BOOST_CHECK(p1 != p3); + } } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END()
\ No newline at end of file diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index af9080b5e9..b55f1c72b1 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,83 +45,71 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } } while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 11)) { - case 0: { - addr_man.Clear(); - break; - } - case 1: { - addr_man.ResolveCollisions(); - break; - } - case 2: { - (void)addr_man.SelectTriedCollision(); - break; - } - case 3: { - (void)addr_man.Select(fuzzed_data_provider.ConsumeBool()); - break; - } - case 4: { - (void)addr_man.GetAddr(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - break; - } - case 5: { - const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); - if (opt_address && opt_net_addr) { - addr_man.Add(*opt_address, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); - } - break; - } - case 6: { - std::vector<CAddress> addresses; - while (fuzzed_data_provider.ConsumeBool()) { + CallOneOf( + fuzzed_data_provider, + [&] { + addr_man.Clear(); + }, + [&] { + addr_man.ResolveCollisions(); + }, + [&] { + (void)addr_man.SelectTriedCollision(); + }, + [&] { + (void)addr_man.Select(fuzzed_data_provider.ConsumeBool()); + }, + [&] { + (void)addr_man.GetAddr(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + }, + [&] { const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - if (!opt_address) { - break; + const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); + if (opt_address && opt_net_addr) { + addr_man.Add(*opt_address, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); } - addresses.push_back(*opt_address); - } - const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); - if (opt_net_addr) { - addr_man.Add(addresses, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); - } - break; - } - case 7: { - const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); - if (opt_service) { - addr_man.Good(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); - } - break; - } - case 8: { - const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); - if (opt_service) { - addr_man.Attempt(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); - } - break; - } - case 9: { - const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); - if (opt_service) { - addr_man.Connected(*opt_service, ConsumeTime(fuzzed_data_provider)); - } - break; - } - case 10: { - const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); - if (opt_service) { - addr_man.SetServices(*opt_service, ServiceFlags{fuzzed_data_provider.ConsumeIntegral<uint64_t>()}); - } - break; - } - case 11: { - (void)addr_man.Check(); - break; - } - } + }, + [&] { + std::vector<CAddress> addresses; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!opt_address) { + break; + } + addresses.push_back(*opt_address); + } + const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); + if (opt_net_addr) { + addr_man.Add(addresses, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); + } + }, + [&] { + const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (opt_service) { + addr_man.Good(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); + } + }, + [&] { + const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (opt_service) { + addr_man.Attempt(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); + } + }, + [&] { + const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (opt_service) { + addr_man.Connected(*opt_service, ConsumeTime(fuzzed_data_provider)); + } + }, + [&] { + const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (opt_service) { + addr_man.SetServices(*opt_service, ServiceFlags{fuzzed_data_provider.ConsumeIntegral<uint64_t>()}); + } + }, + [&] { + (void)addr_man.Check(); + }); } (void)addr_man.size(); CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index eb3424ef28..dbc0b5ab81 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,43 +21,37 @@ FUZZ_TARGET(autofile) FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); CAutoFile auto_file = fuzzed_auto_file_provider.open(); while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { - case 0: { - std::array<uint8_t, 4096> arr{}; - try { - auto_file.read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - } catch (const std::ios_base::failure&) { - } - break; - } - case 1: { - const std::array<uint8_t, 4096> arr{}; - try { - auto_file.write((const char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - } catch (const std::ios_base::failure&) { - } - break; - } - case 2: { - try { - auto_file.ignore(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - } catch (const std::ios_base::failure&) { - } - break; - } - case 3: { - auto_file.fclose(); - break; - } - case 4: { - ReadFromStream(fuzzed_data_provider, auto_file); - break; - } - case 5: { - WriteToStream(fuzzed_data_provider, auto_file); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + std::array<uint8_t, 4096> arr{}; + try { + auto_file.read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + }, + [&] { + const std::array<uint8_t, 4096> arr{}; + try { + auto_file.write((const char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + }, + [&] { + try { + auto_file.ignore(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + }, + [&] { + auto_file.fclose(); + }, + [&] { + ReadFromStream(fuzzed_data_provider, auto_file); + }, + [&] { + WriteToStream(fuzzed_data_provider, auto_file); + }); } (void)auto_file.Get(); (void)auto_file.GetType(); diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index cf69fa0722..124439559e 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,7 +26,7 @@ int64_t ConsumeBanTimeOffset(FuzzedDataProvider& fuzzed_data_provider) noexcept void initialize_banman() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET_INIT(banman, initialize_banman) @@ -38,51 +38,43 @@ FUZZ_TARGET_INIT(banman, initialize_banman) { BanMan ban_man{banlist_file, nullptr, ConsumeBanTimeOffset(fuzzed_data_provider)}; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 11)) { - case 0: { - ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); - break; - } - case 1: { - ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); - break; - } - case 2: { - ban_man.ClearBanned(); - break; - } - case 4: { - ban_man.IsBanned(ConsumeNetAddr(fuzzed_data_provider)); - break; - } - case 5: { - ban_man.IsBanned(ConsumeSubNet(fuzzed_data_provider)); - break; - } - case 6: { - ban_man.Unban(ConsumeNetAddr(fuzzed_data_provider)); - break; - } - case 7: { - ban_man.Unban(ConsumeSubNet(fuzzed_data_provider)); - break; - } - case 8: { - banmap_t banmap; - ban_man.GetBanned(banmap); - break; - } - case 9: { - ban_man.DumpBanlist(); - break; - } - case 11: { - ban_man.Discourage(ConsumeNetAddr(fuzzed_data_provider)); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + }, + [&] { + ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), + ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + }, + [&] { + ban_man.ClearBanned(); + }, + [] {}, + [&] { + ban_man.IsBanned(ConsumeNetAddr(fuzzed_data_provider)); + }, + [&] { + ban_man.IsBanned(ConsumeSubNet(fuzzed_data_provider)); + }, + [&] { + ban_man.Unban(ConsumeNetAddr(fuzzed_data_provider)); + }, + [&] { + ban_man.Unban(ConsumeSubNet(fuzzed_data_provider)); + }, + [&] { + banmap_t banmap; + ban_man.GetBanned(banmap); + }, + [&] { + ban_man.DumpBanlist(); + }, + [] {}, + [&] { + ban_man.Discourage(ConsumeNetAddr(fuzzed_data_provider)); + }); } } fs::remove(banlist_file); diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index c0c66c564b..c5bb8744a4 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,47 +25,43 @@ FUZZ_TARGET(bloom_filter) fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 3)) { - case 0: { - const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); - (void)bloom_filter.contains(b); - bloom_filter.insert(b); - const bool present = bloom_filter.contains(b); - assert(present); - break; - } - case 1: { - const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); - if (!out_point) { - break; - } - (void)bloom_filter.contains(*out_point); - bloom_filter.insert(*out_point); - const bool present = bloom_filter.contains(*out_point); - assert(present); - break; - } - case 2: { - const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); - if (!u256) { - break; - } - (void)bloom_filter.contains(*u256); - bloom_filter.insert(*u256); - const bool present = bloom_filter.contains(*u256); - assert(present); - break; - } - case 3: { - const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - if (!mut_tx) { - break; - } - const CTransaction tx{*mut_tx}; - (void)bloom_filter.IsRelevantAndUpdate(tx); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + (void)bloom_filter.contains(b); + bloom_filter.insert(b); + const bool present = bloom_filter.contains(b); + assert(present); + }, + [&] { + const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!out_point) { + return; + } + (void)bloom_filter.contains(*out_point); + bloom_filter.insert(*out_point); + const bool present = bloom_filter.contains(*out_point); + assert(present); + }, + [&] { + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + if (!u256) { + return; + } + (void)bloom_filter.contains(*u256); + bloom_filter.insert(*u256); + const bool present = bloom_filter.contains(*u256); + assert(present); + }, + [&] { + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mut_tx) { + return; + } + const CTransaction tx{*mut_tx}; + (void)bloom_filter.IsRelevantAndUpdate(tx); + }); (void)bloom_filter.IsWithinSizeConstraints(); } } diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index 23e197456a..ffe38f10fc 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,41 +31,36 @@ FUZZ_TARGET(buffered_file) if (opt_buffered_file && fuzzed_file != nullptr) { bool setpos_fail = false; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 4)) { - case 0: { - std::array<uint8_t, 4096> arr{}; - try { - opt_buffered_file->read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - } catch (const std::ios_base::failure&) { - } - break; - } - case 1: { - opt_buffered_file->SetLimit(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096)); - break; - } - case 2: { - if (!opt_buffered_file->SetPos(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096))) { - setpos_fail = true; - } - break; - } - case 3: { - if (setpos_fail) { - // Calling FindByte(...) after a failed SetPos(...) call may result in an infinite loop. - break; - } - try { - opt_buffered_file->FindByte(fuzzed_data_provider.ConsumeIntegral<char>()); - } catch (const std::ios_base::failure&) { - } - break; - } - case 4: { - ReadFromStream(fuzzed_data_provider, *opt_buffered_file); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + std::array<uint8_t, 4096> arr{}; + try { + opt_buffered_file->read((char*)arr.data(), fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + } catch (const std::ios_base::failure&) { + } + }, + [&] { + opt_buffered_file->SetLimit(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096)); + }, + [&] { + if (!opt_buffered_file->SetPos(fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096))) { + setpos_fail = true; + } + }, + [&] { + if (setpos_fail) { + // Calling FindByte(...) after a failed SetPos(...) call may result in an infinite loop. + return; + } + try { + opt_buffered_file->FindByte(fuzzed_data_provider.ConsumeIntegral<char>()); + } catch (const std::ios_base::failure&) { + } + }, + [&] { + ReadFromStream(fuzzed_data_provider, *opt_buffered_file); + }); } opt_buffered_file->GetPos(); opt_buffered_file->GetType(); diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 1ae421493e..19486365f6 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,9 +36,7 @@ bool operator==(const Coin& a, const Coin& b) void initialize_coins_view() { - static const ECCVerifyHandle ecc_verify_handle; - ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); } FUZZ_TARGET_INIT(coins_view, initialize_coins_view) @@ -50,103 +48,93 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) Coin random_coin; CMutableTransaction random_mutable_transaction; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 9)) { - case 0: { - if (random_coin.IsSpent()) { - break; - } - Coin coin = random_coin; - bool expected_code_path = false; - const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); - try { - coins_view_cache.AddCoin(random_out_point, std::move(coin), possible_overwrite); - expected_code_path = true; - } catch (const std::logic_error& e) { - if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { - assert(!possible_overwrite); + CallOneOf( + fuzzed_data_provider, + [&] { + if (random_coin.IsSpent()) { + return; + } + Coin coin = random_coin; + bool expected_code_path = false; + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + coins_view_cache.AddCoin(random_out_point, std::move(coin), possible_overwrite); expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } } - } - assert(expected_code_path); - break; - } - case 1: { - (void)coins_view_cache.Flush(); - break; - } - case 2: { - coins_view_cache.SetBestBlock(ConsumeUInt256(fuzzed_data_provider)); - break; - } - case 3: { - Coin move_to; - (void)coins_view_cache.SpendCoin(random_out_point, fuzzed_data_provider.ConsumeBool() ? &move_to : nullptr); - break; - } - case 4: { - coins_view_cache.Uncache(random_out_point); - break; - } - case 5: { - if (fuzzed_data_provider.ConsumeBool()) { - backend_coins_view = CCoinsView{}; - } - coins_view_cache.SetBackend(backend_coins_view); - break; - } - case 6: { - const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); - if (!opt_out_point) { - break; - } - random_out_point = *opt_out_point; - break; - } - case 7: { - const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); - if (!opt_coin) { - break; - } - random_coin = *opt_coin; - break; - } - case 8: { - const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - if (!opt_mutable_transaction) { - break; - } - random_mutable_transaction = *opt_mutable_transaction; - break; - } - case 9: { - CCoinsMap coins_map; - while (fuzzed_data_provider.ConsumeBool()) { - CCoinsCacheEntry coins_cache_entry; - coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); + assert(expected_code_path); + }, + [&] { + (void)coins_view_cache.Flush(); + }, + [&] { + coins_view_cache.SetBestBlock(ConsumeUInt256(fuzzed_data_provider)); + }, + [&] { + Coin move_to; + (void)coins_view_cache.SpendCoin(random_out_point, fuzzed_data_provider.ConsumeBool() ? &move_to : nullptr); + }, + [&] { + coins_view_cache.Uncache(random_out_point); + }, + [&] { if (fuzzed_data_provider.ConsumeBool()) { - coins_cache_entry.coin = random_coin; - } else { - const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); - if (!opt_coin) { - break; + backend_coins_view = CCoinsView{}; + } + coins_view_cache.SetBackend(backend_coins_view); + }, + [&] { + const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!opt_out_point) { + return; + } + random_out_point = *opt_out_point; + }, + [&] { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + return; + } + random_coin = *opt_coin; + }, + [&] { + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_mutable_transaction) { + return; + } + random_mutable_transaction = *opt_mutable_transaction; + }, + [&] { + CCoinsMap coins_map; + while (fuzzed_data_provider.ConsumeBool()) { + CCoinsCacheEntry coins_cache_entry; + coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); + if (fuzzed_data_provider.ConsumeBool()) { + coins_cache_entry.coin = random_coin; + } else { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + return; + } + coins_cache_entry.coin = *opt_coin; } - coins_cache_entry.coin = *opt_coin; + coins_map.emplace(random_out_point, std::move(coins_cache_entry)); } - coins_map.emplace(random_out_point, std::move(coins_cache_entry)); - } - bool expected_code_path = false; - try { - coins_view_cache.BatchWrite(coins_map, fuzzed_data_provider.ConsumeBool() ? ConsumeUInt256(fuzzed_data_provider) : coins_view_cache.GetBestBlock()); - expected_code_path = true; - } catch (const std::logic_error& e) { - if (e.what() == std::string{"FRESH flag misapplied to coin that exists in parent cache"}) { + bool expected_code_path = false; + try { + coins_view_cache.BatchWrite(coins_map, fuzzed_data_provider.ConsumeBool() ? ConsumeUInt256(fuzzed_data_provider) : coins_view_cache.GetBestBlock()); expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"FRESH flag misapplied to coin that exists in parent cache"}) { + expected_code_path = true; + } } - } - assert(expected_code_path); - break; - } - } + assert(expected_code_path); + }); } { @@ -199,97 +187,90 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) } if (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { - case 0: { - const CTransaction transaction{random_mutable_transaction}; - bool is_spent = false; - for (const CTxOut& tx_out : transaction.vout) { - if (Coin{tx_out, 0, transaction.IsCoinBase()}.IsSpent()) { - is_spent = true; + CallOneOf( + fuzzed_data_provider, + [&] { + const CTransaction transaction{random_mutable_transaction}; + bool is_spent = false; + for (const CTxOut& tx_out : transaction.vout) { + if (Coin{tx_out, 0, transaction.IsCoinBase()}.IsSpent()) { + is_spent = true; + } + } + if (is_spent) { + // Avoid: + // coins.cpp:69: void CCoinsViewCache::AddCoin(const COutPoint &, Coin &&, bool): Assertion `!coin.IsSpent()' failed. + return; } - } - if (is_spent) { - // Avoid: - // coins.cpp:69: void CCoinsViewCache::AddCoin(const COutPoint &, Coin &&, bool): Assertion `!coin.IsSpent()' failed. - break; - } - bool expected_code_path = false; - const int height = fuzzed_data_provider.ConsumeIntegral<int>(); - const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); - try { - AddCoins(coins_view_cache, transaction, height, possible_overwrite); - expected_code_path = true; - } catch (const std::logic_error& e) { - if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { - assert(!possible_overwrite); + bool expected_code_path = false; + const int height = fuzzed_data_provider.ConsumeIntegral<int>(); + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + AddCoins(coins_view_cache, transaction, height, possible_overwrite); expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } } - } - assert(expected_code_path); - break; - } - case 1: { - (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, false); - (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, true); - break; - } - case 2: { - TxValidationState state; - CAmount tx_fee_out; - const CTransaction transaction{random_mutable_transaction}; - if (ContainsSpentInput(transaction, coins_view_cache)) { - // Avoid: - // consensus/tx_verify.cpp:171: bool Consensus::CheckTxInputs(const CTransaction &, TxValidationState &, const CCoinsViewCache &, int, CAmount &): Assertion `!coin.IsSpent()' failed. - break; - } - try { - (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); - assert(MoneyRange(tx_fee_out)); - } catch (const std::runtime_error&) { - } - break; - } - case 3: { - const CTransaction transaction{random_mutable_transaction}; - if (ContainsSpentInput(transaction, coins_view_cache)) { - // Avoid: - // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. - break; - } - (void)GetP2SHSigOpCount(transaction, coins_view_cache); - break; - } - case 4: { - const CTransaction transaction{random_mutable_transaction}; - if (ContainsSpentInput(transaction, coins_view_cache)) { - // Avoid: - // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. - break; - } - const int flags = fuzzed_data_provider.ConsumeIntegral<int>(); - if (!transaction.vin.empty() && (flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { - // Avoid: - // script/interpreter.cpp:1705: size_t CountWitnessSigOps(const CScript &, const CScript &, const CScriptWitness *, unsigned int): Assertion `(flags & SCRIPT_VERIFY_P2SH) != 0' failed. - break; - } - (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); - break; - } - case 5: { - CCoinsStats stats; - bool expected_code_path = false; - try { - (void)GetUTXOStats(&coins_view_cache, stats, CoinStatsHashType::HASH_SERIALIZED); - } catch (const std::logic_error&) { - expected_code_path = true; - } - assert(expected_code_path); - break; - } - case 6: { - (void)IsWitnessStandard(CTransaction{random_mutable_transaction}, coins_view_cache); - break; - } - } + assert(expected_code_path); + }, + [&] { + (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, false); + (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, true); + }, + [&] { + TxValidationState state; + CAmount tx_fee_out; + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:171: bool Consensus::CheckTxInputs(const CTransaction &, TxValidationState &, const CCoinsViewCache &, int, CAmount &): Assertion `!coin.IsSpent()' failed. + return; + } + try { + (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); + assert(MoneyRange(tx_fee_out)); + } catch (const std::runtime_error&) { + } + }, + [&] { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + return; + } + (void)GetP2SHSigOpCount(transaction, coins_view_cache); + }, + [&] { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + return; + } + const int flags = fuzzed_data_provider.ConsumeIntegral<int>(); + if (!transaction.vin.empty() && (flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { + // Avoid: + // script/interpreter.cpp:1705: size_t CountWitnessSigOps(const CScript &, const CScript &, const CScriptWitness *, unsigned int): Assertion `(flags & SCRIPT_VERIFY_P2SH) != 0' failed. + return; + } + (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); + }, + [&] { + CCoinsStats stats; + bool expected_code_path = false; + try { + (void)GetUTXOStats(&coins_view_cache, stats, CoinStatsHashType::HASH_SERIALIZED); + } catch (const std::logic_error&) { + expected_code_path = true; + } + assert(expected_code_path); + }, + [&] { + (void)IsWitnessStandard(CTransaction{random_mutable_transaction}, coins_view_cache); + }); } } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index fdf51d8558..3fb8d8ca06 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,7 +17,7 @@ void initialize_connman() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET_INIT(connman, initialize_connman) @@ -32,108 +32,104 @@ FUZZ_TARGET_INIT(connman, initialize_connman) CSubNet random_subnet; std::string random_string; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 27)) { - case 0: - random_address = ConsumeAddress(fuzzed_data_provider); - break; - case 1: - random_netaddr = ConsumeNetAddr(fuzzed_data_provider); - break; - case 2: - random_service = ConsumeService(fuzzed_data_provider); - break; - case 3: - random_subnet = ConsumeSubNet(fuzzed_data_provider); - break; - case 4: - random_string = fuzzed_data_provider.ConsumeRandomLengthString(64); - break; - case 5: { - std::vector<CAddress> addresses; - while (fuzzed_data_provider.ConsumeBool()) { - addresses.push_back(ConsumeAddress(fuzzed_data_provider)); - } - // Limit nTimePenalty to int32_t to avoid signed integer overflow - (void)connman.AddNewAddresses(addresses, ConsumeAddress(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()); - break; - } - case 6: - connman.AddNode(random_string); - break; - case 7: - connman.CheckIncomingNonce(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - case 8: - connman.DisconnectNode(fuzzed_data_provider.ConsumeIntegral<NodeId>()); - break; - case 9: - connman.DisconnectNode(random_netaddr); - break; - case 10: - connman.DisconnectNode(random_string); - break; - case 11: - connman.DisconnectNode(random_subnet); - break; - case 12: - connman.ForEachNode([](auto) {}); - break; - case 13: - connman.ForEachNodeThen([](auto) {}, []() {}); - break; - case 14: - (void)connman.ForNode(fuzzed_data_provider.ConsumeIntegral<NodeId>(), [&](auto) { return fuzzed_data_provider.ConsumeBool(); }); - break; - case 15: - (void)connman.GetAddresses(fuzzed_data_provider.ConsumeIntegral<size_t>(), fuzzed_data_provider.ConsumeIntegral<size_t>()); - break; - case 16: { - (void)connman.GetAddresses(random_node, fuzzed_data_provider.ConsumeIntegral<size_t>(), fuzzed_data_provider.ConsumeIntegral<size_t>()); - break; - } - case 17: - (void)connman.GetDeterministicRandomizer(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - case 18: - (void)connman.GetNodeCount(fuzzed_data_provider.PickValueInArray({CConnman::CONNECTIONS_NONE, CConnman::CONNECTIONS_IN, CConnman::CONNECTIONS_OUT, CConnman::CONNECTIONS_ALL})); - break; - case 19: - connman.MarkAddressGood(random_address); - break; - case 20: - (void)connman.OutboundTargetReached(fuzzed_data_provider.ConsumeBool()); - break; - case 21: - // Limit now to int32_t to avoid signed integer overflow - (void)connman.PoissonNextSendInbound(fuzzed_data_provider.ConsumeIntegral<int32_t>(), fuzzed_data_provider.ConsumeIntegral<int>()); - break; - case 22: { - CSerializedNetMsg serialized_net_msg; - serialized_net_msg.m_type = fuzzed_data_provider.ConsumeRandomLengthString(CMessageHeader::COMMAND_SIZE); - serialized_net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - connman.PushMessage(&random_node, std::move(serialized_net_msg)); - break; - } - case 23: - connman.RemoveAddedNode(random_string); - break; - case 24: { - const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (SanityCheckASMap(asmap)) { - connman.SetAsmap(asmap); - } - break; - } - case 25: - connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); - break; - case 26: - connman.SetServices(random_service, ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS)); - break; - case 27: - connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); - break; - } + CallOneOf( + fuzzed_data_provider, + [&] { + random_address = ConsumeAddress(fuzzed_data_provider); + }, + [&] { + random_netaddr = ConsumeNetAddr(fuzzed_data_provider); + }, + [&] { + random_service = ConsumeService(fuzzed_data_provider); + }, + [&] { + random_subnet = ConsumeSubNet(fuzzed_data_provider); + }, + [&] { + random_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + }, + [&] { + std::vector<CAddress> addresses; + while (fuzzed_data_provider.ConsumeBool()) { + addresses.push_back(ConsumeAddress(fuzzed_data_provider)); + } + // Limit nTimePenalty to int32_t to avoid signed integer overflow + (void)connman.AddNewAddresses(addresses, ConsumeAddress(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()); + }, + [&] { + connman.AddNode(random_string); + }, + [&] { + connman.CheckIncomingNonce(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + connman.DisconnectNode(fuzzed_data_provider.ConsumeIntegral<NodeId>()); + }, + [&] { + connman.DisconnectNode(random_netaddr); + }, + [&] { + connman.DisconnectNode(random_string); + }, + [&] { + connman.DisconnectNode(random_subnet); + }, + [&] { + connman.ForEachNode([](auto) {}); + }, + [&] { + connman.ForEachNodeThen([](auto) {}, []() {}); + }, + [&] { + (void)connman.ForNode(fuzzed_data_provider.ConsumeIntegral<NodeId>(), [&](auto) { return fuzzed_data_provider.ConsumeBool(); }); + }, + [&] { + (void)connman.GetAddresses(fuzzed_data_provider.ConsumeIntegral<size_t>(), fuzzed_data_provider.ConsumeIntegral<size_t>()); + }, + [&] { + (void)connman.GetAddresses(random_node, fuzzed_data_provider.ConsumeIntegral<size_t>(), fuzzed_data_provider.ConsumeIntegral<size_t>()); + }, + [&] { + (void)connman.GetDeterministicRandomizer(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + (void)connman.GetNodeCount(fuzzed_data_provider.PickValueInArray({CConnman::CONNECTIONS_NONE, CConnman::CONNECTIONS_IN, CConnman::CONNECTIONS_OUT, CConnman::CONNECTIONS_ALL})); + }, + [&] { + connman.MarkAddressGood(random_address); + }, + [&] { + (void)connman.OutboundTargetReached(fuzzed_data_provider.ConsumeBool()); + }, + [&] { + // Limit now to int32_t to avoid signed integer overflow + (void)connman.PoissonNextSendInbound(fuzzed_data_provider.ConsumeIntegral<int32_t>(), fuzzed_data_provider.ConsumeIntegral<int>()); + }, + [&] { + CSerializedNetMsg serialized_net_msg; + serialized_net_msg.m_type = fuzzed_data_provider.ConsumeRandomLengthString(CMessageHeader::COMMAND_SIZE); + serialized_net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + connman.PushMessage(&random_node, std::move(serialized_net_msg)); + }, + [&] { + connman.RemoveAddedNode(random_string); + }, + [&] { + const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (SanityCheckASMap(asmap)) { + connman.SetAsmap(asmap); + } + }, + [&] { + connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); + }, + [&] { + connman.SetServices(random_service, ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS)); + }, + [&] { + connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); + }); } (void)connman.GetAddedNodeInfo(); (void)connman.GetExtraFullOutboundCount(); diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp index 9668b84e7b..eeeac18968 100644 --- a/src/test/fuzz/crypto.cpp +++ b/src/test/fuzz/crypto.cpp @@ -1,10 +1,9 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> -#include <crypto/muhash.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -36,112 +35,86 @@ FUZZ_TARGET(crypto) CSHA512 sha512; SHA3_256 sha3; CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; - MuHash3072 muhash; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { - case 0: { - if (fuzzed_data_provider.ConsumeBool()) { - data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - if (data.empty()) { - data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + CallOneOf( + fuzzed_data_provider, + [&] { + if (fuzzed_data_provider.ConsumeBool()) { + data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } } - } - (void)hash160.Write(data); - (void)hash256.Write(data); - (void)hmac_sha256.Write(data.data(), data.size()); - (void)hmac_sha512.Write(data.data(), data.size()); - (void)ripemd160.Write(data.data(), data.size()); - (void)sha1.Write(data.data(), data.size()); - (void)sha256.Write(data.data(), data.size()); - (void)sha3.Write(data); - (void)sha512.Write(data.data(), data.size()); - (void)sip_hasher.Write(data.data(), data.size()); + (void)hash160.Write(data); + (void)hash256.Write(data); + (void)hmac_sha256.Write(data.data(), data.size()); + (void)hmac_sha512.Write(data.data(), data.size()); + (void)ripemd160.Write(data.data(), data.size()); + (void)sha1.Write(data.data(), data.size()); + (void)sha256.Write(data.data(), data.size()); + (void)sha3.Write(data); + (void)sha512.Write(data.data(), data.size()); + (void)sip_hasher.Write(data.data(), data.size()); - (void)Hash(data); - (void)Hash160(data); - (void)sha512.Size(); - - if (fuzzed_data_provider.ConsumeBool()) { - muhash *= MuHash3072(data); - } else { - muhash /= MuHash3072(data); - } - break; - } - case 1: { - (void)hash160.Reset(); - (void)hash256.Reset(); - (void)ripemd160.Reset(); - (void)sha1.Reset(); - (void)sha256.Reset(); - (void)sha3.Reset(); - (void)sha512.Reset(); - muhash = MuHash3072(); - break; - } - case 2: { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)) { - case 0: { - data.resize(CHash160::OUTPUT_SIZE); - hash160.Finalize(data); - break; - } - case 1: { - data.resize(CHash256::OUTPUT_SIZE); - hash256.Finalize(data); - break; - } - case 2: { - data.resize(CHMAC_SHA256::OUTPUT_SIZE); - hmac_sha256.Finalize(data.data()); - break; - } - case 3: { - data.resize(CHMAC_SHA512::OUTPUT_SIZE); - hmac_sha512.Finalize(data.data()); - break; - } - case 4: { - data.resize(CRIPEMD160::OUTPUT_SIZE); - ripemd160.Finalize(data.data()); - break; - } - case 5: { - data.resize(CSHA1::OUTPUT_SIZE); - sha1.Finalize(data.data()); - break; - } - case 6: { - data.resize(CSHA256::OUTPUT_SIZE); - sha256.Finalize(data.data()); - break; - } - case 7: { - data.resize(CSHA512::OUTPUT_SIZE); - sha512.Finalize(data.data()); - break; - } - case 8: { - data.resize(1); - data[0] = sip_hasher.Finalize() % 256; - break; - } - case 9: { - data.resize(SHA3_256::OUTPUT_SIZE); - sha3.Finalize(data); - break; - } - case 10: { - uint256 out; - muhash.Finalize(out); - break; - } - } - break; - } - } + (void)Hash(data); + (void)Hash160(data); + (void)sha512.Size(); + }, + [&] { + (void)hash160.Reset(); + (void)hash256.Reset(); + (void)ripemd160.Reset(); + (void)sha1.Reset(); + (void)sha256.Reset(); + (void)sha3.Reset(); + (void)sha512.Reset(); + }, + [&] { + CallOneOf( + fuzzed_data_provider, + [&] { + data.resize(CHash160::OUTPUT_SIZE); + hash160.Finalize(data); + }, + [&] { + data.resize(CHash256::OUTPUT_SIZE); + hash256.Finalize(data); + }, + [&] { + data.resize(CHMAC_SHA256::OUTPUT_SIZE); + hmac_sha256.Finalize(data.data()); + }, + [&] { + data.resize(CHMAC_SHA512::OUTPUT_SIZE); + hmac_sha512.Finalize(data.data()); + }, + [&] { + data.resize(CRIPEMD160::OUTPUT_SIZE); + ripemd160.Finalize(data.data()); + }, + [&] { + data.resize(CSHA1::OUTPUT_SIZE); + sha1.Finalize(data.data()); + }, + [&] { + data.resize(CSHA256::OUTPUT_SIZE); + sha256.Finalize(data.data()); + }, + [&] { + data.resize(CSHA512::OUTPUT_SIZE); + sha512.Finalize(data.data()); + }, + [&] { + data.resize(1); + data[0] = sip_hasher.Finalize() % 256; + }, + [&] { + data.resize(SHA3_256::OUTPUT_SIZE); + sha3.Finalize(data); + }); + }); } if (fuzzed_data_provider.ConsumeBool()) { uint64_t state[25]; diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index d751466f11..8adfa92420 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,31 +20,26 @@ FUZZ_TARGET(crypto_chacha20) chacha20 = ChaCha20{key.data(), key.size()}; } while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { - case 0: { - const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); - chacha20.SetKey(key.data(), key.size()); - break; - } - case 1: { - chacha20.SetIV(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - } - case 2: { - chacha20.Seek(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - } - case 3: { - std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - chacha20.Keystream(output.data(), output.size()); - break; - } - case 4: { - std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); - chacha20.Crypt(input.data(), output.data(), input.size()); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); + chacha20.SetKey(key.data(), key.size()); + }, + [&] { + chacha20.SetIV(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + chacha20.Seek(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + chacha20.Keystream(output.data(), output.size()); + }, + [&] { + std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); + chacha20.Crypt(input.data(), output.data(), input.size()); + }); } } diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index 631af9c70d..bb4ef22158 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,44 +29,43 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead) std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); bool is_encrypt = fuzzed_data_provider.ConsumeBool(); while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { - case 0: { - buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); - in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); - out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); - break; - } - case 1: { - (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); - break; - } - case 2: { - uint32_t len = 0; - const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); - assert(ok); - break; - } - case 3: { - seqnr_payload += 1; - aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; - if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { - aad_pos = 0; - seqnr_aad += 1; - } - break; - } - case 4: { - seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>(); - break; - } - case 5: { - seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>(); - break; - } - case 6: { - is_encrypt = fuzzed_data_provider.ConsumeBool(); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + buffer_size = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(64, 4096); + in = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + out = std::vector<uint8_t>(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); + }, + [&] { + (void)aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffer_size, is_encrypt); + }, + [&] { + uint32_t len = 0; + const bool ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); + assert(ok); + }, + [&] { + if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) { + return; + } + seqnr_payload += 1; + aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; + if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { + aad_pos = 0; + if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) { + return; + } + seqnr_aad += 1; + } + }, + [&] { + seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + }, + [&] { + seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + }, + [&] { + is_encrypt = fuzzed_data_provider.ConsumeBool(); + }); } } diff --git a/src/test/fuzz/data_stream.cpp b/src/test/fuzz/data_stream.cpp index 28fc528ceb..d6ef0c6691 100644 --- a/src/test/fuzz/data_stream.cpp +++ b/src/test/fuzz/data_stream.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,7 @@ void initialize_data_stream_addr_man() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET_INIT(data_stream_addr_man, initialize_data_stream_addr_man) diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 74dec6475e..64c6e49615 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,8 +30,6 @@ #include <stdint.h> #include <unistd.h> -#include <vector> - #include <test/fuzz/fuzz.h> void initialize_deserialize() @@ -71,7 +69,7 @@ T Deserialize(CDataStream ds) } template <typename T> -void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj, const Optional<int> protocol_version = nullopt) +void DeserializeFromFuzzingInput(FuzzBufferType buffer, T& obj, const Optional<int> protocol_version = nullopt) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); if (protocol_version) { diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index fd87667755..1fab46ff13 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,15 +13,15 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>>& FuzzTargets() +std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>>& FuzzTargets() { - static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>> g_fuzz_targets; + static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>> g_fuzz_targets; return g_fuzz_targets; } -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init) +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden) { - const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init)); + const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init), hidden); Assert(it_ins.second); } @@ -31,6 +31,7 @@ void initialize() { if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { + if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } Assert(false); @@ -43,7 +44,7 @@ void initialize() std::get<1>(it->second)(); } -#if defined(PROVIDE_MAIN_FUNCTION) +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -59,8 +60,7 @@ static bool read_stdin(std::vector<uint8_t>& data) extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { static const auto& test_one_input = *Assert(g_test_one_input); - const std::vector<uint8_t> input(data, data + size); - test_one_input(input); + test_one_input({data, size}); return 0; } @@ -71,8 +71,8 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -#if defined(PROVIDE_MAIN_FUNCTION) -__attribute__((weak)) int main(int argc, char** argv) +#if defined(PROVIDE_FUZZ_MAIN_FUNCTION) +int main(int argc, char** argv) { initialize(); static const auto& test_one_input = *Assert(g_test_one_input); diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 52841e069a..2bad77bdc1 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -1,33 +1,40 @@ -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H +#include <span.h> + #include <cstdint> #include <functional> #include <string_view> -#include <vector> -using TypeTestOneInput = std::function<void(const std::vector<uint8_t>&)>; +using FuzzBufferType = Span<const uint8_t>; + +using TypeTestOneInput = std::function<void(FuzzBufferType)>; using TypeInitialize = std::function<void()>; +using TypeHidden = bool; -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init); +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden); -inline void FuzzFrameworkEmptyFun() {} +inline void FuzzFrameworkEmptyInitFun() {} #define FUZZ_TARGET(name) \ - FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyFun) - -#define FUZZ_TARGET_INIT(name, init_fun) \ - void name##_fuzz_target(const std::vector<uint8_t>&); \ - struct name##_Before_Main { \ - name##_Before_Main() \ - { \ - FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun); \ - } \ - } const static g_##name##_before_main; \ - void name##_fuzz_target(const std::vector<uint8_t>& buffer) + FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyInitFun) + +#define FUZZ_TARGET_INIT(name, init_fun) \ + FUZZ_TARGET_INIT_HIDDEN(name, init_fun, false) + +#define FUZZ_TARGET_INIT_HIDDEN(name, init_fun, hidden) \ + void name##_fuzz_target(FuzzBufferType); \ + struct name##_Before_Main { \ + name##_Before_Main() \ + { \ + FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun, hidden); \ + } \ + } const static g_##name##_before_main; \ + void name##_fuzz_target(FuzzBufferType buffer) #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp index 4dfb2e3da0..908e9a1c83 100644 --- a/src/test/fuzz/kitchen_sink.cpp +++ b/src/test/fuzz/kitchen_sink.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,12 +28,6 @@ constexpr TransactionError ALL_TRANSACTION_ERROR[] = { TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED, }; - -constexpr FeeEstimateHorizon ALL_FEE_EST_HORIZON[] = { - FeeEstimateHorizon::SHORT_HALFLIFE, - FeeEstimateHorizon::MED_HALFLIFE, - FeeEstimateHorizon::LONG_HALFLIFE, -}; }; // namespace // The fuzzing kitchen sink: Fuzzing harness for functions that need to be @@ -48,7 +42,7 @@ FUZZ_TARGET(kitchen_sink) (void)RPCErrorFromTransactionError(transaction_error); (void)TransactionErrorString(transaction_error); - (void)StringForFeeEstimateHorizon(fuzzed_data_provider.PickValueInArray(ALL_FEE_EST_HORIZON)); + (void)StringForFeeEstimateHorizon(fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS)); const OutputType output_type = fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES); const std::string& output_type_string = FormatOutputType(output_type); diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index c428a86631..dbd0c76d42 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ void initialize_load_external_block_file() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); } FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) @@ -27,5 +27,5 @@ FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) return; } FlatFilePos flat_file_pos; - LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); + ::ChainstateActive().LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); } diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 15bcfab3ad..1eefd4c521 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,33 +17,31 @@ FUZZ_TARGET(merkleblock) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CPartialMerkleTree partial_merkle_tree; - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1)) { - case 0: { - const std::optional<CPartialMerkleTree> opt_partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); - if (opt_partial_merkle_tree) { - partial_merkle_tree = *opt_partial_merkle_tree; - } - break; - } - case 1: { - CMerkleBlock merkle_block; - const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); - CBloomFilter bloom_filter; - std::set<uint256> txids; - if (opt_block && !opt_block->vtx.empty()) { - if (fuzzed_data_provider.ConsumeBool()) { - merkle_block = CMerkleBlock{*opt_block, bloom_filter}; - } else if (fuzzed_data_provider.ConsumeBool()) { - while (fuzzed_data_provider.ConsumeBool()) { - txids.insert(ConsumeUInt256(fuzzed_data_provider)); + CallOneOf( + fuzzed_data_provider, + [&] { + const std::optional<CPartialMerkleTree> opt_partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); + if (opt_partial_merkle_tree) { + partial_merkle_tree = *opt_partial_merkle_tree; + } + }, + [&] { + CMerkleBlock merkle_block; + const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + CBloomFilter bloom_filter; + std::set<uint256> txids; + if (opt_block && !opt_block->vtx.empty()) { + if (fuzzed_data_provider.ConsumeBool()) { + merkle_block = CMerkleBlock{*opt_block, bloom_filter}; + } else if (fuzzed_data_provider.ConsumeBool()) { + while (fuzzed_data_provider.ConsumeBool()) { + txids.insert(ConsumeUInt256(fuzzed_data_provider)); + } + merkle_block = CMerkleBlock{*opt_block, txids}; } - merkle_block = CMerkleBlock{*opt_block, txids}; } - } - partial_merkle_tree = merkle_block.txn; - break; - } - } + partial_merkle_tree = merkle_block.txn; + }); (void)partial_merkle_tree.GetNumTransactions(); std::vector<uint256> matches; std::vector<unsigned int> indices; diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 87359c9d39..4ea9511870 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(muhash) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); @@ -41,6 +41,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) muhash.Finalize(out2); assert(out == out2); + MuHash3072 muhash3; + muhash3 *= muhash; + uint256 out3; + muhash3.Finalize(out3); + assert(out == out3); // Test that removing all added elements brings the object back to it's initial state muhash /= muhash; @@ -50,4 +55,9 @@ void test_one_input(const std::vector<uint8_t>& buffer) muhash2.Finalize(out2); assert(out == out2); + + muhash3.Remove(data); + muhash3.Remove(data2); + muhash3.Finalize(out3); + assert(out == out3); } diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 3ca921b5cf..b056f46f2e 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,7 +22,7 @@ void initialize_net() { - static const BasicTestingSetup basic_testing_setup; + static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::MAIN); } FUZZ_TARGET_INIT(net, initialize_net) @@ -32,85 +32,74 @@ FUZZ_TARGET_INIT(net, initialize_net) CNode node{ConsumeNode(fuzzed_data_provider)}; node.SetCommonVersion(fuzzed_data_provider.ConsumeIntegral<int>()); while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)) { - case 0: { - node.CloseSocketDisconnect(); - break; - } - case 1: { - node.MaybeSetAddrName(fuzzed_data_provider.ConsumeRandomLengthString(32)); - break; - } - case 2: { - const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(asmap)) { - break; - } - CNodeStats stats; - node.copyStats(stats, asmap); - break; - } - case 3: { - const CNode* add_ref_node = node.AddRef(); - assert(add_ref_node == &node); - break; - } - case 4: { - if (node.GetRefCount() > 0) { - node.Release(); - } - break; - } - case 5: { - if (node.m_addr_known == nullptr) { - break; - } - const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - if (!addr_opt) { - break; - } - node.AddAddressKnown(*addr_opt); - break; - } - case 6: { - if (node.m_addr_known == nullptr) { - break; - } - const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - if (!addr_opt) { - break; - } - FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; - node.PushAddress(*addr_opt, fast_random_context); - break; - } - case 7: { - const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); - if (!inv_opt) { - break; - } - node.AddKnownTx(inv_opt->hash); - break; - } - case 8: { - node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); - break; - } - case 9: { - const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); - if (!service_opt) { - break; - } - node.SetAddrLocal(*service_opt); - break; - } - case 10: { - const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); - bool complete; - node.ReceiveMsgBytes(b, complete); - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + node.CloseSocketDisconnect(); + }, + [&] { + node.MaybeSetAddrName(fuzzed_data_provider.ConsumeRandomLengthString(32)); + }, + [&] { + const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (!SanityCheckASMap(asmap)) { + return; + } + CNodeStats stats; + node.copyStats(stats, asmap); + }, + [&] { + const CNode* add_ref_node = node.AddRef(); + assert(add_ref_node == &node); + }, + [&] { + if (node.GetRefCount() > 0) { + node.Release(); + } + }, + [&] { + if (node.m_addr_known == nullptr) { + return; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + return; + } + node.AddAddressKnown(*addr_opt); + }, + [&] { + if (node.m_addr_known == nullptr) { + return; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + return; + } + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + node.PushAddress(*addr_opt, fast_random_context); + }, + [&] { + const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); + if (!inv_opt) { + return; + } + node.AddKnownTx(inv_opt->hash); + }, + [&] { + node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); + }, + [&] { + const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (!service_opt) { + return; + } + node.SetAddrLocal(*service_opt); + }, + [&] { + const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + bool complete; + node.ReceiveMsgBytes(b, complete); + }); } (void)node.GetAddrLocal(); diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 6e9bb47ff6..f9d8129ca9 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,6 @@ #include <cassert> #include <cstdint> -#include <netinet/in.h> #include <vector> FUZZ_TARGET(netaddress) diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 8a17a4b51b..311550f041 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,7 +16,7 @@ void initialize_policy_estimator() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) @@ -24,52 +24,48 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CBlockPolicyEstimator block_policy_estimator; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { - case 0: { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - if (!mtx) { - break; - } - const CTransaction tx{*mtx}; - block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); - if (fuzzed_data_provider.ConsumeBool()) { - (void)block_policy_estimator.removeTx(tx.GetHash(), /* inBlock */ fuzzed_data_provider.ConsumeBool()); - } - break; - } - case 1: { - std::vector<CTxMemPoolEntry> mempool_entries; - while (fuzzed_data_provider.ConsumeBool()) { + CallOneOf( + fuzzed_data_provider, + [&] { const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { - break; + return; } const CTransaction tx{*mtx}; - mempool_entries.push_back(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); - } - std::vector<const CTxMemPoolEntry*> ptrs; - ptrs.reserve(mempool_entries.size()); - for (const CTxMemPoolEntry& mempool_entry : mempool_entries) { - ptrs.push_back(&mempool_entry); - } - block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); - break; - } - case 2: { - (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /* inBlock */ fuzzed_data_provider.ConsumeBool()); - break; - } - case 3: { - block_policy_estimator.FlushUnconfirmed(); - break; - } - } + block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + (void)block_policy_estimator.removeTx(tx.GetHash(), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + } + }, + [&] { + std::vector<CTxMemPoolEntry> mempool_entries; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + break; + } + const CTransaction tx{*mtx}; + mempool_entries.push_back(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } + std::vector<const CTxMemPoolEntry*> ptrs; + ptrs.reserve(mempool_entries.size()); + for (const CTxMemPoolEntry& mempool_entry : mempool_entries) { + ptrs.push_back(&mempool_entry); + } + block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); + }, + [&] { + (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + }, + [&] { + block_policy_estimator.FlushUnconfirmed(); + }); (void)block_policy_estimator.estimateFee(fuzzed_data_provider.ConsumeIntegral<int>()); EstimationResult result; - (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); + (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); FeeCalculation fee_calculation; (void)block_policy_estimator.estimateSmartFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeBool() ? &fee_calculation : nullptr, fuzzed_data_provider.ConsumeBool()); - (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE})); + (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS)); } { FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 8fa52143d8..c24ad3d49a 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ void initialize_policy_estimator_io() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeNoLogFileContext<>(); } FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 02beb6eb37..53726ca893 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -43,7 +43,10 @@ FUZZ_TARGET_INIT(pow, initialize_pow) current_block.nHeight = current_height; } if (fuzzed_data_provider.ConsumeBool()) { - current_block.nTime = fixed_time + current_height * consensus_params.nPowTargetSpacing; + const uint32_t seconds = current_height * consensus_params.nPowTargetSpacing; + if (!AdditionOverflow(fixed_time, seconds)) { + current_block.nTime = fixed_time + seconds; + } } if (fuzzed_data_provider.ConsumeBool()) { current_block.nBits = fixed_bits; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index b6865c5347..e3571e15b7 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,46 +30,63 @@ #include <iostream> #include <memory> #include <string> -#include <vector> namespace { const TestingSetup* g_setup; } // namespace +size_t& GetNumMsgTypes() +{ + static size_t g_num_msg_types{0}; + return g_num_msg_types; +} +#define FUZZ_TARGET_MSG(msg_type) \ + struct msg_type##_Count_Before_Main { \ + msg_type##_Count_Before_Main() \ + { \ + ++GetNumMsgTypes(); \ + } \ + } const static g_##msg_type##_count_before_main; \ + FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ + { \ + fuzz_target(buffer, #msg_type); \ + } + void initialize_process_message() { - static TestingSetup setup{ - CBaseChainParams::REGTEST, - { - "-nodebuglogfile", - }, - }; - g_setup = &setup; + Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); } SyncWithValidationInterfaceQueue(); } -void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + SetMockTime(1610000000); // any time to successfully reset ibd chainstate.ResetIbd(); + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } - const bool jump_out_of_ibd{fuzzed_data_provider.ConsumeBool()}; - if (jump_out_of_ibd) chainstate.JumpOutOfIbd(); CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); - p2p_node.fSuccessfullyConnected = true; - p2p_node.nVersion = PROTOCOL_VERSION; - p2p_node.SetCommonVersion(PROTOCOL_VERSION); + + const bool successfully_connected{fuzzed_data_provider.ConsumeBool()}; + p2p_node.fSuccessfullyConnected = successfully_connected; connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); + + const auto mock_time = ConsumeTime(fuzzed_data_provider); + SetMockTime(mock_time); // fuzzed_data_provider is fully consumed after this call, don't use it CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; @@ -88,27 +105,37 @@ void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO } FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_INIT(process_message_addr, initialize_process_message) { fuzz_target(buffer, "addr"); } -FUZZ_TARGET_INIT(process_message_block, initialize_process_message) { fuzz_target(buffer, "block"); } -FUZZ_TARGET_INIT(process_message_blocktxn, initialize_process_message) { fuzz_target(buffer, "blocktxn"); } -FUZZ_TARGET_INIT(process_message_cmpctblock, initialize_process_message) { fuzz_target(buffer, "cmpctblock"); } -FUZZ_TARGET_INIT(process_message_feefilter, initialize_process_message) { fuzz_target(buffer, "feefilter"); } -FUZZ_TARGET_INIT(process_message_filteradd, initialize_process_message) { fuzz_target(buffer, "filteradd"); } -FUZZ_TARGET_INIT(process_message_filterclear, initialize_process_message) { fuzz_target(buffer, "filterclear"); } -FUZZ_TARGET_INIT(process_message_filterload, initialize_process_message) { fuzz_target(buffer, "filterload"); } -FUZZ_TARGET_INIT(process_message_getaddr, initialize_process_message) { fuzz_target(buffer, "getaddr"); } -FUZZ_TARGET_INIT(process_message_getblocks, initialize_process_message) { fuzz_target(buffer, "getblocks"); } -FUZZ_TARGET_INIT(process_message_getblocktxn, initialize_process_message) { fuzz_target(buffer, "getblocktxn"); } -FUZZ_TARGET_INIT(process_message_getdata, initialize_process_message) { fuzz_target(buffer, "getdata"); } -FUZZ_TARGET_INIT(process_message_getheaders, initialize_process_message) { fuzz_target(buffer, "getheaders"); } -FUZZ_TARGET_INIT(process_message_headers, initialize_process_message) { fuzz_target(buffer, "headers"); } -FUZZ_TARGET_INIT(process_message_inv, initialize_process_message) { fuzz_target(buffer, "inv"); } -FUZZ_TARGET_INIT(process_message_mempool, initialize_process_message) { fuzz_target(buffer, "mempool"); } -FUZZ_TARGET_INIT(process_message_notfound, initialize_process_message) { fuzz_target(buffer, "notfound"); } -FUZZ_TARGET_INIT(process_message_ping, initialize_process_message) { fuzz_target(buffer, "ping"); } -FUZZ_TARGET_INIT(process_message_pong, initialize_process_message) { fuzz_target(buffer, "pong"); } -FUZZ_TARGET_INIT(process_message_sendcmpct, initialize_process_message) { fuzz_target(buffer, "sendcmpct"); } -FUZZ_TARGET_INIT(process_message_sendheaders, initialize_process_message) { fuzz_target(buffer, "sendheaders"); } -FUZZ_TARGET_INIT(process_message_tx, initialize_process_message) { fuzz_target(buffer, "tx"); } -FUZZ_TARGET_INIT(process_message_verack, initialize_process_message) { fuzz_target(buffer, "verack"); } -FUZZ_TARGET_INIT(process_message_version, initialize_process_message) { fuzz_target(buffer, "version"); } +FUZZ_TARGET_MSG(addr); +FUZZ_TARGET_MSG(addrv2); +FUZZ_TARGET_MSG(block); +FUZZ_TARGET_MSG(blocktxn); +FUZZ_TARGET_MSG(cfcheckpt); +FUZZ_TARGET_MSG(cfheaders); +FUZZ_TARGET_MSG(cfilter); +FUZZ_TARGET_MSG(cmpctblock); +FUZZ_TARGET_MSG(feefilter); +FUZZ_TARGET_MSG(filteradd); +FUZZ_TARGET_MSG(filterclear); +FUZZ_TARGET_MSG(filterload); +FUZZ_TARGET_MSG(getaddr); +FUZZ_TARGET_MSG(getblocks); +FUZZ_TARGET_MSG(getblocktxn); +FUZZ_TARGET_MSG(getcfcheckpt); +FUZZ_TARGET_MSG(getcfheaders); +FUZZ_TARGET_MSG(getcfilters); +FUZZ_TARGET_MSG(getdata); +FUZZ_TARGET_MSG(getheaders); +FUZZ_TARGET_MSG(headers); +FUZZ_TARGET_MSG(inv); +FUZZ_TARGET_MSG(mempool); +FUZZ_TARGET_MSG(merkleblock); +FUZZ_TARGET_MSG(notfound); +FUZZ_TARGET_MSG(ping); +FUZZ_TARGET_MSG(pong); +FUZZ_TARGET_MSG(sendaddrv2); +FUZZ_TARGET_MSG(sendcmpct); +FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(tx); +FUZZ_TARGET_MSG(verack); +FUZZ_TARGET_MSG(version); +FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index fcfa623508..f62a0c64ed 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,18 +17,14 @@ #include <validation.h> #include <validationinterface.h> +namespace { const TestingSetup* g_setup; +} // namespace void initialize_process_messages() { - static TestingSetup setup{ - CBaseChainParams::REGTEST, - { - "-nodebuglogfile", - }, - }; - g_setup = &setup; - + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); } @@ -41,29 +37,30 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + SetMockTime(1610000000); // any time to successfully reset ibd chainstate.ResetIbd(); - std::vector<CNode*> peers; - bool jump_out_of_ibd{false}; + std::vector<CNode*> peers; const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); for (int i = 0; i < num_peers_to_add; ++i) { peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); - p2p_node.fSuccessfullyConnected = true; + const bool successfully_connected{fuzzed_data_provider.ConsumeBool()}; + p2p_node.fSuccessfullyConnected = successfully_connected; p2p_node.fPauseSend = false; - p2p_node.nVersion = PROTOCOL_VERSION; - p2p_node.SetCommonVersion(PROTOCOL_VERSION); g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); connman.AddTestNode(p2p_node); } while (fuzzed_data_provider.ConsumeBool()) { - if (!jump_out_of_ibd) jump_out_of_ibd = fuzzed_data_provider.ConsumeBool(); - if (jump_out_of_ibd && chainstate.IsInitialBlockDownload()) chainstate.JumpOutOfIbd(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + const auto mock_time = ConsumeTime(fuzzed_data_provider); + SetMockTime(mock_time); + CSerializedNetMsg net_msg; net_msg.m_type = random_message_type; net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 6087ee964a..07059cce76 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,29 +22,27 @@ FUZZ_TARGET(rolling_bloom_filter) fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 1000), 0.999 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max())}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 2)) { - case 0: { - const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); - (void)rolling_bloom_filter.contains(b); - rolling_bloom_filter.insert(b); - const bool present = rolling_bloom_filter.contains(b); - assert(present); - break; - } - case 1: { - const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); - if (!u256) { - break; - } - (void)rolling_bloom_filter.contains(*u256); - rolling_bloom_filter.insert(*u256); - const bool present = rolling_bloom_filter.contains(*u256); - assert(present); - break; - } - case 2: - rolling_bloom_filter.reset(); - break; - } + CallOneOf( + fuzzed_data_provider, + [&] { + const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + (void)rolling_bloom_filter.contains(b); + rolling_bloom_filter.insert(b); + const bool present = rolling_bloom_filter.contains(b); + assert(present); + }, + [&] { + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + if (!u256) { + return; + } + (void)rolling_bloom_filter.contains(*u256); + rolling_bloom_filter.insert(*u256); + const bool present = rolling_bloom_filter.contains(*u256); + assert(present); + }, + [&] { + rolling_bloom_filter.reset(); + }); } } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index d883426c81..193862e847 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -154,13 +154,13 @@ FUZZ_TARGET_INIT(script, initialize_script) { WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_1.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_2.length = witness_unknown_program_2.size(); std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 2091ad5d91..5f07acbcc7 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,12 +28,12 @@ // // (normal build) // $ mkdir dump -// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot --dumptests; done +// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot.py --dumptests; done // $ ... // -// (fuzz test build) +// (libFuzzer build) // $ mkdir dump-min -// $ ./src/test/fuzz/script_assets_test_minimizer -merge=1 dump-min/ dump/ +// $ FUZZ=script_assets_test_minimizer ./src/test/fuzz/fuzz -merge=1 -use_value_profile=1 dump-min/ dump/ // $ (echo -en '[\n'; cat dump-min/* | head -c -2; echo -en '\n]') >script_assets_test.json namespace { @@ -190,7 +190,7 @@ ECCVerifyHandle handle; } // namespace -FUZZ_TARGET(script_assets_test_minimizer) +FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, FuzzFrameworkEmptyInitFun, /* hidden */ true) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index d232e984bc..eb1c808a88 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,56 +16,53 @@ FUZZ_TARGET(script_ops) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScript script = ConsumeScript(fuzzed_data_provider); while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { - case 0: { - CScript s = ConsumeScript(fuzzed_data_provider); - script = std::move(s); - break; - } - case 1: { - const CScript& s = ConsumeScript(fuzzed_data_provider); - script = s; - break; - } - case 2: - script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); - break; - case 3: - script << ConsumeOpcodeType(fuzzed_data_provider); - break; - case 4: - script << ConsumeScriptNum(fuzzed_data_provider); - break; - case 5: - script << ConsumeRandomLengthByteVector(fuzzed_data_provider); - break; - case 6: - script.clear(); - break; - case 7: { - (void)script.GetSigOpCount(false); - (void)script.GetSigOpCount(true); - (void)script.GetSigOpCount(script); - (void)script.HasValidOps(); - (void)script.IsPayToScriptHash(); - (void)script.IsPayToWitnessScriptHash(); - (void)script.IsPushOnly(); - (void)script.IsUnspendable(); - { - CScript::const_iterator pc = script.begin(); - opcodetype opcode; - (void)script.GetOp(pc, opcode); - std::vector<uint8_t> data; - (void)script.GetOp(pc, opcode, data); - (void)script.IsPushOnly(pc); - } - { - int version; - std::vector<uint8_t> program; - (void)script.IsWitnessProgram(version, program); - } - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + CScript s = ConsumeScript(fuzzed_data_provider); + script = std::move(s); + }, + [&] { + const CScript& s = ConsumeScript(fuzzed_data_provider); + script = s; + }, + [&] { + script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }, + [&] { + script << ConsumeOpcodeType(fuzzed_data_provider); + }, + [&] { + script << ConsumeScriptNum(fuzzed_data_provider); + }, + [&] { + script << ConsumeRandomLengthByteVector(fuzzed_data_provider); + }, + [&] { + script.clear(); + }, + [&] { + (void)script.GetSigOpCount(false); + (void)script.GetSigOpCount(true); + (void)script.GetSigOpCount(script); + (void)script.HasValidOps(); + (void)script.IsPayToScriptHash(); + (void)script.IsPayToWitnessScriptHash(); + (void)script.IsPushOnly(); + (void)script.IsUnspendable(); + { + CScript::const_iterator pc = script.begin(); + opcodetype opcode; + (void)script.GetOp(pc, opcode); + std::vector<uint8_t> data; + (void)script.GetOp(pc, opcode, data); + (void)script.IsPushOnly(pc); + } + { + int version; + std::vector<uint8_t> program; + (void)script.IsWitnessProgram(version, program); + } + }); } } diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 650318f13c..62ed50d13f 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,105 +29,99 @@ FUZZ_TARGET(scriptnum_ops) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScriptNum script_num = ConsumeScriptNum(fuzzed_data_provider); while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 11)) { - case 0: { - const int64_t i = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - assert((script_num == i) != (script_num != i)); - assert((script_num <= i) != (script_num > i)); - assert((script_num >= i) != (script_num < i)); - // Avoid signed integer overflow: - // script/script.h:264:93: runtime error: signed integer overflow: -2261405121394637306 + -9223372036854775802 cannot be represented in type 'long' - if (IsValidAddition(script_num, CScriptNum{i})) { - assert((script_num + i) - i == script_num); - } - // Avoid signed integer overflow: - // script/script.h:265:93: runtime error: signed integer overflow: 9223371895120855039 - -9223372036854710486 cannot be represented in type 'long' - if (IsValidSubtraction(script_num, CScriptNum{i})) { - assert((script_num - i) + i == script_num); - } - break; - } - case 1: { - const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); - assert((script_num == random_script_num) != (script_num != random_script_num)); - assert((script_num <= random_script_num) != (script_num > random_script_num)); - assert((script_num >= random_script_num) != (script_num < random_script_num)); - // Avoid signed integer overflow: - // script/script.h:264:93: runtime error: signed integer overflow: -9223126527765971126 + -9223372036854756825 cannot be represented in type 'long' - if (IsValidAddition(script_num, random_script_num)) { - assert((script_num + random_script_num) - random_script_num == script_num); - } - // Avoid signed integer overflow: - // script/script.h:265:93: runtime error: signed integer overflow: 6052837899185946624 - -9223372036854775808 cannot be represented in type 'long' - if (IsValidSubtraction(script_num, random_script_num)) { - assert((script_num - random_script_num) + random_script_num == script_num); - } - break; - } - case 2: { - const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); - if (!IsValidAddition(script_num, random_script_num)) { - // Avoid assertion failure: - // ./script/script.h:292: CScriptNum &CScriptNum::operator+=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value <= std::numeric_limits<int64_t>::max() - rhs) || (rhs < 0 && m_value >= std::numeric_limits<int64_t>::min() - rhs)' failed. - break; - } - script_num += random_script_num; - break; - } - case 3: { - const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); - if (!IsValidSubtraction(script_num, random_script_num)) { - // Avoid assertion failure: - // ./script/script.h:300: CScriptNum &CScriptNum::operator-=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value >= std::numeric_limits<int64_t>::min() + rhs) || (rhs < 0 && m_value <= std::numeric_limits<int64_t>::max() + rhs)' failed. - break; - } - script_num -= random_script_num; - break; - } - case 4: - script_num = script_num & fuzzed_data_provider.ConsumeIntegral<int64_t>(); - break; - case 5: - script_num = script_num & ConsumeScriptNum(fuzzed_data_provider); - break; - case 6: - script_num &= ConsumeScriptNum(fuzzed_data_provider); - break; - case 7: - if (script_num == CScriptNum{std::numeric_limits<int64_t>::min()}) { - // Avoid assertion failure: - // ./script/script.h:279: CScriptNum CScriptNum::operator-() const: Assertion `m_value != std::numeric_limits<int64_t>::min()' failed. - break; - } - script_num = -script_num; - break; - case 8: - script_num = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - break; - case 9: { - const int64_t random_integer = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - if (!IsValidAddition(script_num, CScriptNum{random_integer})) { - // Avoid assertion failure: - // ./script/script.h:292: CScriptNum &CScriptNum::operator+=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value <= std::numeric_limits<int64_t>::max() - rhs) || (rhs < 0 && m_value >= std::numeric_limits<int64_t>::min() - rhs)' failed. - break; - } - script_num += random_integer; - break; - } - case 10: { - const int64_t random_integer = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - if (!IsValidSubtraction(script_num, CScriptNum{random_integer})) { - // Avoid assertion failure: - // ./script/script.h:300: CScriptNum &CScriptNum::operator-=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value >= std::numeric_limits<int64_t>::min() + rhs) || (rhs < 0 && m_value <= std::numeric_limits<int64_t>::max() + rhs)' failed. - break; - } - script_num -= random_integer; - break; - } - case 11: - script_num &= fuzzed_data_provider.ConsumeIntegral<int64_t>(); - break; - } + CallOneOf( + fuzzed_data_provider, + [&] { + const int64_t i = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + assert((script_num == i) != (script_num != i)); + assert((script_num <= i) != (script_num > i)); + assert((script_num >= i) != (script_num < i)); + // Avoid signed integer overflow: + // script/script.h:264:93: runtime error: signed integer overflow: -2261405121394637306 + -9223372036854775802 cannot be represented in type 'long' + if (IsValidAddition(script_num, CScriptNum{i})) { + assert((script_num + i) - i == script_num); + } + // Avoid signed integer overflow: + // script/script.h:265:93: runtime error: signed integer overflow: 9223371895120855039 - -9223372036854710486 cannot be represented in type 'long' + if (IsValidSubtraction(script_num, CScriptNum{i})) { + assert((script_num - i) + i == script_num); + } + }, + [&] { + const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); + assert((script_num == random_script_num) != (script_num != random_script_num)); + assert((script_num <= random_script_num) != (script_num > random_script_num)); + assert((script_num >= random_script_num) != (script_num < random_script_num)); + // Avoid signed integer overflow: + // script/script.h:264:93: runtime error: signed integer overflow: -9223126527765971126 + -9223372036854756825 cannot be represented in type 'long' + if (IsValidAddition(script_num, random_script_num)) { + assert((script_num + random_script_num) - random_script_num == script_num); + } + // Avoid signed integer overflow: + // script/script.h:265:93: runtime error: signed integer overflow: 6052837899185946624 - -9223372036854775808 cannot be represented in type 'long' + if (IsValidSubtraction(script_num, random_script_num)) { + assert((script_num - random_script_num) + random_script_num == script_num); + } + }, + [&] { + const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); + if (!IsValidAddition(script_num, random_script_num)) { + // Avoid assertion failure: + // ./script/script.h:292: CScriptNum &CScriptNum::operator+=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value <= std::numeric_limits<int64_t>::max() - rhs) || (rhs < 0 && m_value >= std::numeric_limits<int64_t>::min() - rhs)' failed. + return; + } + script_num += random_script_num; + }, + [&] { + const CScriptNum random_script_num = ConsumeScriptNum(fuzzed_data_provider); + if (!IsValidSubtraction(script_num, random_script_num)) { + // Avoid assertion failure: + // ./script/script.h:300: CScriptNum &CScriptNum::operator-=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value >= std::numeric_limits<int64_t>::min() + rhs) || (rhs < 0 && m_value <= std::numeric_limits<int64_t>::max() + rhs)' failed. + return; + } + script_num -= random_script_num; + }, + [&] { + script_num = script_num & fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }, + [&] { + script_num = script_num & ConsumeScriptNum(fuzzed_data_provider); + }, + [&] { + script_num &= ConsumeScriptNum(fuzzed_data_provider); + }, + [&] { + if (script_num == CScriptNum{std::numeric_limits<int64_t>::min()}) { + // Avoid assertion failure: + // ./script/script.h:279: CScriptNum CScriptNum::operator-() const: Assertion `m_value != std::numeric_limits<int64_t>::min()' failed. + return; + } + script_num = -script_num; + }, + [&] { + script_num = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }, + [&] { + const int64_t random_integer = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + if (!IsValidAddition(script_num, CScriptNum{random_integer})) { + // Avoid assertion failure: + // ./script/script.h:292: CScriptNum &CScriptNum::operator+=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value <= std::numeric_limits<int64_t>::max() - rhs) || (rhs < 0 && m_value >= std::numeric_limits<int64_t>::min() - rhs)' failed. + return; + } + script_num += random_integer; + }, + [&] { + const int64_t random_integer = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + if (!IsValidSubtraction(script_num, CScriptNum{random_integer})) { + // Avoid assertion failure: + // ./script/script.h:300: CScriptNum &CScriptNum::operator-=(const int64_t &): Assertion `rhs == 0 || (rhs > 0 && m_value >= std::numeric_limits<int64_t>::min() + rhs) || (rhs < 0 && m_value <= std::numeric_limits<int64_t>::max() + rhs)' failed. + return; + } + script_num -= random_integer; + }, + [&] { + script_num &= fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }); (void)script_num.getint(); (void)script_num.getvch(); } diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 541322d484..0d1999731f 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,7 +17,7 @@ void initialize_signet() { - InitializeFuzzingContext(CBaseChainParams::SIGNET); + static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::SIGNET); } FUZZ_TARGET_INIT(signet, initialize_signet) diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 282a2cd8ca..93b4948a2f 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -67,6 +67,7 @@ FUZZ_TARGET(string) } OutputType output_type; (void)ParseOutputType(random_string_1, output_type); + (void)RemovePrefix(random_string_1, random_string_2); (void)ResolveErrMsg(random_string_1, random_string_2); try { (void)RPCConvertNamedValues(random_string_1, random_string_vector); @@ -78,7 +79,9 @@ FUZZ_TARGET(string) } (void)SanitizeString(random_string_1); (void)SanitizeString(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)); +#ifndef WIN32 (void)ShellEscape(random_string_1); +#endif // WIN32 int port_out; std::string host_out; SplitHostPort(random_string_1, port_out, host_out); diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp index 4af0e750ce..2c92b159a5 100644 --- a/src/test/fuzz/strprintf.cpp +++ b/src/test/fuzz/strprintf.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <tinyformat.h> #include <util/strencodings.h> #include <util/translation.h> @@ -109,32 +110,32 @@ FUZZ_TARGET(str_printf) } try { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 5)) { - case 0: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); - break; - case 1: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); - break; - case 2: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); - break; - case 3: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); - break; - case 4: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<char>()); - break; - case 5: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeBool()); - break; - } + CallOneOf( + fuzzed_data_provider, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<char>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeBool()); + }); } catch (const tinyformat::format_error&) { } @@ -155,40 +156,40 @@ FUZZ_TARGET(str_printf) } try { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { - case 0: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); - break; - case 1: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); - break; - case 2: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); - break; - case 3: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); - break; - case 4: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); - break; - case 5: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); - break; - case 6: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); - break; - case 7: - (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - } + CallOneOf( + fuzzed_data_provider, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + }, + [&] { + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }); } catch (const tinyformat::format_error&) { } } diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 375a8c1ed0..d9571209fa 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -32,71 +32,63 @@ FUZZ_TARGET(system) } while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 7)) { - case 0: { - args_manager.SelectConfigNetwork(fuzzed_data_provider.ConsumeRandomLengthString(16)); - break; - } - case 1: { - args_manager.SoftSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); - break; - } - case 2: { - args_manager.ForceSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); - break; - } - case 3: { - args_manager.SoftSetBoolArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeBool()); - break; - } - case 4: { - const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); - // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. - const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); - if (args_manager.GetArgFlags(argument_name) != nullopt) { - break; - } - args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); - break; - } - case 5: { - // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. - const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); - std::vector<std::string> hidden_arguments; - for (const std::string& name : names) { - const std::string hidden_argument = GetArgumentName(name); - if (args_manager.GetArgFlags(hidden_argument) != nullopt) { - continue; + CallOneOf( + fuzzed_data_provider, + [&] { + args_manager.SelectConfigNetwork(fuzzed_data_provider.ConsumeRandomLengthString(16)); + }, + [&] { + args_manager.SoftSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + }, + [&] { + args_manager.ForceSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + }, + [&] { + args_manager.SoftSetBoolArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeBool()); + }, + [&] { + const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); + if (args_manager.GetArgFlags(argument_name) != nullopt) { + return; } - if (std::find(hidden_arguments.begin(), hidden_arguments.end(), hidden_argument) != hidden_arguments.end()) { - continue; + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>() & ~ArgsManager::COMMAND, options_category); + }, + [&] { + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<std::string> hidden_arguments; + for (const std::string& name : names) { + const std::string hidden_argument = GetArgumentName(name); + if (args_manager.GetArgFlags(hidden_argument) != nullopt) { + continue; + } + if (std::find(hidden_arguments.begin(), hidden_arguments.end(), hidden_argument) != hidden_arguments.end()) { + continue; + } + hidden_arguments.push_back(hidden_argument); } - hidden_arguments.push_back(hidden_argument); - } - args_manager.AddHiddenArgs(hidden_arguments); - break; - } - case 6: { - args_manager.ClearArgs(); - break; - } - case 7: { - const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); - std::vector<const char*> argv; - argv.reserve(random_arguments.size()); - for (const std::string& random_argument : random_arguments) { - argv.push_back(random_argument.c_str()); - } - try { - std::string error; - (void)args_manager.ParseParameters(argv.size(), argv.data(), error); - } catch (const std::logic_error&) { - } - break; - } - } + args_manager.AddHiddenArgs(hidden_arguments); + }, + [&] { + args_manager.ClearArgs(); + }, + [&] { + const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<const char*> argv; + argv.reserve(random_arguments.size()); + for (const std::string& random_argument : random_arguments) { + argv.push_back(random_argument.c_str()); + } + try { + std::string error; + (void)args_manager.ParseParameters(argv.size(), argv.data(), error); + } catch (const std::logic_error&) { + } + }); } const std::string s1 = fuzzed_data_provider.ConsumeRandomLengthString(16); diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp new file mode 100644 index 0000000000..0a541e4186 --- /dev/null +++ b/src/test/fuzz/util.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util.h> +#include <version.h> + +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept +{ + const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); + const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); + const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); + const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + + node.nServices = remote_services; + node.m_permissionFlags = permission_flags; + if (init_version) { + node.nVersion = version; + node.SetCommonVersion(std::min(version, PROTOCOL_VERSION)); + } + if (node.m_tx_relay != nullptr) { + LOCK(node.m_tx_relay->cs_filter); + node.m_tx_relay->fRelayTxes = filter_txs; + } +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 94c691936e..f3bc3c78ab 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,6 +27,7 @@ #include <txmempool.h> #include <uint256.h> #include <util/time.h> +#include <util/vector.h> #include <version.h> #include <algorithm> @@ -36,6 +37,17 @@ #include <string> #include <vector> +template <typename... Callables> +void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) +{ + constexpr size_t call_size{sizeof...(callables)}; + static_assert(call_size >= 1); + const size_t call_index{fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, call_size - 1)}; + + size_t i{0}; + return ((i++ == call_index ? callables() : void()), ...); +} + [[nodiscard]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { const std::string s = fuzzed_data_provider.ConsumeRandomLengthString(max_length); @@ -49,7 +61,7 @@ [[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { - return {ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; + return CDataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; } [[nodiscard]] inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept @@ -165,37 +177,31 @@ template <typename WeakEnumType, size_t size> [[nodiscard]] inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept { CTxDestination tx_destination; - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { - case 0: { - tx_destination = CNoDestination{}; - break; - } - case 1: { - tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; - break; - } - case 2: { - tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; - break; - } - case 3: { - tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; - break; - } - case 4: { - tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; - break; - } - case 5: { - WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<int>(); - const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); - tx_destination = witness_unknown; - break; - } - } + CallOneOf( + fuzzed_data_provider, + [&] { + tx_destination = CNoDestination{}; + }, + [&] { + tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + WitnessUnknown witness_unknown{}; + witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); + witness_unknown.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); + tx_destination = witness_unknown; + }); return tx_destination; } @@ -263,8 +269,8 @@ inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcep const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); CNetAddr net_addr; if (network == Network::NET_IPV4) { - const in_addr v4_addr = { - .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); net_addr = CNetAddr{v4_addr}; } else if (network == Network::NET_IPV6) { if (fuzzed_data_provider.remaining_bytes() >= 16) { @@ -296,7 +302,7 @@ inline CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcep } template <bool ReturnUniquePtr = false> -auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = nullopt) noexcept +auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept { const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegral<NodeId>()); const ServiceFlags local_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); @@ -314,11 +320,21 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N return CNode{node_id, local_services, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion}; } } -inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = nullopt) { return ConsumeNode<true>(fdp, node_id_in); } +inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } -inline void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept; + +template <class T = const BasicTestingSetup> +std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseChainParams::REGTEST, const std::vector<const char*>& extra_args = {}) { - static const BasicTestingSetup basic_testing_setup{chain_name, {"-nodebuglogfile"}}; + // Prepend default arguments for fuzzing + const std::vector<const char*> arguments = Cat( + { + "-nodebuglogfile", + }, + extra_args); + + return MakeUnique<T>(chain_name, arguments); } class FuzzedFileProvider @@ -337,32 +353,26 @@ public: return nullptr; } std::string mode; - switch (m_fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { - case 0: { - mode = "r"; - break; - } - case 1: { - mode = "r+"; - break; - } - case 2: { - mode = "w"; - break; - } - case 3: { - mode = "w+"; - break; - } - case 4: { - mode = "a"; - break; - } - case 5: { - mode = "a+"; - break; - } - } + CallOneOf( + m_fuzzed_data_provider, + [&] { + mode = "r"; + }, + [&] { + mode = "r+"; + }, + [&] { + mode = "w"; + }, + [&] { + mode = "w+"; + }, + [&] { + mode = "a"; + }, + [&] { + mode = "a+"; + }); #ifdef _GNU_SOURCE const cookie_io_functions_t io_hooks = { FuzzedFileProvider::read, @@ -460,66 +470,64 @@ public: return {fuzzed_data_provider}; } -#define WRITE_TO_STREAM_CASE(id, type, consume) \ - case id: { \ - type o = consume; \ - stream << o; \ - break; \ +#define WRITE_TO_STREAM_CASE(type, consume) \ + [&] { \ + type o = consume; \ + stream << o; \ } template <typename Stream> void WriteToStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noexcept { while (fuzzed_data_provider.ConsumeBool()) { try { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 13)) { - WRITE_TO_STREAM_CASE(0, bool, fuzzed_data_provider.ConsumeBool()) - WRITE_TO_STREAM_CASE(1, char, fuzzed_data_provider.ConsumeIntegral<char>()) - WRITE_TO_STREAM_CASE(2, int8_t, fuzzed_data_provider.ConsumeIntegral<int8_t>()) - WRITE_TO_STREAM_CASE(3, uint8_t, fuzzed_data_provider.ConsumeIntegral<uint8_t>()) - WRITE_TO_STREAM_CASE(4, int16_t, fuzzed_data_provider.ConsumeIntegral<int16_t>()) - WRITE_TO_STREAM_CASE(5, uint16_t, fuzzed_data_provider.ConsumeIntegral<uint16_t>()) - WRITE_TO_STREAM_CASE(6, int32_t, fuzzed_data_provider.ConsumeIntegral<int32_t>()) - WRITE_TO_STREAM_CASE(7, uint32_t, fuzzed_data_provider.ConsumeIntegral<uint32_t>()) - WRITE_TO_STREAM_CASE(8, int64_t, fuzzed_data_provider.ConsumeIntegral<int64_t>()) - WRITE_TO_STREAM_CASE(9, uint64_t, fuzzed_data_provider.ConsumeIntegral<uint64_t>()) - WRITE_TO_STREAM_CASE(10, float, fuzzed_data_provider.ConsumeFloatingPoint<float>()) - WRITE_TO_STREAM_CASE(11, double, fuzzed_data_provider.ConsumeFloatingPoint<double>()) - WRITE_TO_STREAM_CASE(12, std::string, fuzzed_data_provider.ConsumeRandomLengthString(32)) - WRITE_TO_STREAM_CASE(13, std::vector<char>, ConsumeRandomLengthIntegralVector<char>(fuzzed_data_provider)) - } + CallOneOf( + fuzzed_data_provider, + WRITE_TO_STREAM_CASE(bool, fuzzed_data_provider.ConsumeBool()), + WRITE_TO_STREAM_CASE(char, fuzzed_data_provider.ConsumeIntegral<char>()), + WRITE_TO_STREAM_CASE(int8_t, fuzzed_data_provider.ConsumeIntegral<int8_t>()), + WRITE_TO_STREAM_CASE(uint8_t, fuzzed_data_provider.ConsumeIntegral<uint8_t>()), + WRITE_TO_STREAM_CASE(int16_t, fuzzed_data_provider.ConsumeIntegral<int16_t>()), + WRITE_TO_STREAM_CASE(uint16_t, fuzzed_data_provider.ConsumeIntegral<uint16_t>()), + WRITE_TO_STREAM_CASE(int32_t, fuzzed_data_provider.ConsumeIntegral<int32_t>()), + WRITE_TO_STREAM_CASE(uint32_t, fuzzed_data_provider.ConsumeIntegral<uint32_t>()), + WRITE_TO_STREAM_CASE(int64_t, fuzzed_data_provider.ConsumeIntegral<int64_t>()), + WRITE_TO_STREAM_CASE(uint64_t, fuzzed_data_provider.ConsumeIntegral<uint64_t>()), + WRITE_TO_STREAM_CASE(float, fuzzed_data_provider.ConsumeFloatingPoint<float>()), + WRITE_TO_STREAM_CASE(double, fuzzed_data_provider.ConsumeFloatingPoint<double>()), + WRITE_TO_STREAM_CASE(std::string, fuzzed_data_provider.ConsumeRandomLengthString(32)), + WRITE_TO_STREAM_CASE(std::vector<char>, ConsumeRandomLengthIntegralVector<char>(fuzzed_data_provider))); } catch (const std::ios_base::failure&) { break; } } } -#define READ_FROM_STREAM_CASE(id, type) \ - case id: { \ - type o; \ - stream >> o; \ - break; \ +#define READ_FROM_STREAM_CASE(type) \ + [&] { \ + type o; \ + stream >> o; \ } template <typename Stream> void ReadFromStream(FuzzedDataProvider& fuzzed_data_provider, Stream& stream) noexcept { while (fuzzed_data_provider.ConsumeBool()) { try { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 13)) { - READ_FROM_STREAM_CASE(0, bool) - READ_FROM_STREAM_CASE(1, char) - READ_FROM_STREAM_CASE(2, int8_t) - READ_FROM_STREAM_CASE(3, uint8_t) - READ_FROM_STREAM_CASE(4, int16_t) - READ_FROM_STREAM_CASE(5, uint16_t) - READ_FROM_STREAM_CASE(6, int32_t) - READ_FROM_STREAM_CASE(7, uint32_t) - READ_FROM_STREAM_CASE(8, int64_t) - READ_FROM_STREAM_CASE(9, uint64_t) - READ_FROM_STREAM_CASE(10, float) - READ_FROM_STREAM_CASE(11, double) - READ_FROM_STREAM_CASE(12, std::string) - READ_FROM_STREAM_CASE(13, std::vector<char>) - } + CallOneOf( + fuzzed_data_provider, + READ_FROM_STREAM_CASE(bool), + READ_FROM_STREAM_CASE(char), + READ_FROM_STREAM_CASE(int8_t), + READ_FROM_STREAM_CASE(uint8_t), + READ_FROM_STREAM_CASE(int16_t), + READ_FROM_STREAM_CASE(uint16_t), + READ_FROM_STREAM_CASE(int32_t), + READ_FROM_STREAM_CASE(uint32_t), + READ_FROM_STREAM_CASE(int64_t), + READ_FROM_STREAM_CASE(uint64_t), + READ_FROM_STREAM_CASE(float), + READ_FROM_STREAM_CASE(double), + READ_FROM_STREAM_CASE(std::string), + READ_FROM_STREAM_CASE(std::vector<char>)); } catch (const std::ios_base::failure&) { break; } diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 87f6470afa..41a626c0ea 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -107,14 +107,14 @@ BOOST_AUTO_TEST_CASE(siphash) // Check test vectors from spec, one byte at a time CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); - for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); ++x) + for (uint8_t x=0; x<std::size(siphash_4_2_testvec); ++x) { BOOST_CHECK_EQUAL(hasher2.Finalize(), siphash_4_2_testvec[x]); hasher2.Write(&x, 1); } // Check test vectors from spec, eight bytes at a time CSipHasher hasher3(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); - for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); x+=8) + for (uint8_t x=0; x<std::size(siphash_4_2_testvec); x+=8) { BOOST_CHECK_EQUAL(hasher3.Finalize(), siphash_4_2_testvec[x]); hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)| diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index e967273636..aa628371e6 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -28,7 +28,7 @@ struct MinerTestingSetup : public TestingSetup { void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); bool TestSequenceLocks(const CTransaction& tx, int flags) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) { - return CheckSequenceLocks(*m_node.mempool, tx, flags); + return CheckSequenceLocks(::ChainstateActive(), *m_node.mempool, tx, flags); } BlockAssembler AssemblerForTest(const CChainParams& params); }; @@ -123,6 +123,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co m_node.mempool->addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4U); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashMediumFeeTx); @@ -157,6 +158,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co hashLowFeeTx = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -191,6 +193,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee m_node.mempool->addUnchecked(entry.Fee(10000).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 9U); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } @@ -216,11 +219,10 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // We can't make transactions until we have inputs // Therefore, load 110 blocks :) - static_assert(sizeof(blockinfo) / sizeof(*blockinfo) == 110, "Should have 110 blocks to import"); + static_assert(std::size(blockinfo) == 110, "Should have 110 blocks to import"); int baseheight = 0; std::vector<CTransactionRef> txFirst; - for (unsigned int i = 0; i < sizeof(blockinfo)/sizeof(*blockinfo); ++i) - { + for (const auto& bi : blockinfo) { CBlock *pblock = &pblocktemplate->block; // pointer for convenience { LOCK(cs_main); @@ -229,7 +231,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.nVersion = 1; txCoinbase.vin[0].scriptSig = CScript(); - txCoinbase.vin[0].scriptSig.push_back(blockinfo[i].extranonce); + txCoinbase.vin[0].scriptSig.push_back(bi.extranonce); txCoinbase.vin[0].scriptSig.push_back(::ChainActive().Height()); txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); @@ -239,7 +241,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) if (txFirst.size() < 4) txFirst.push_back(pblock->vtx[0]); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); - pblock->nNonce = blockinfo[i].nonce; + pblock->nNonce = bi.nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); @@ -435,7 +437,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = 0; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(CTransaction(tx), flags)); // Locktime passes + BOOST_CHECK(CheckFinalTx(::ChainActive().Tip(), CTransaction(tx), flags)); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, CreateBlockIndex(::ChainActive().Tip()->nHeight + 2))); // Sequence locks pass on 2nd block @@ -445,7 +447,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 2; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(CheckFinalTx(CTransaction(tx), flags)); // Locktime passes + BOOST_CHECK(CheckFinalTx(::ChainActive().Tip(), CTransaction(tx), flags)); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail for (int i = 0; i < CBlockIndex::nMedianTimeSpan; i++) @@ -461,7 +463,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.nLockTime = ::ChainActive().Tip()->nHeight + 1; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(CTransaction(tx), flags)); // Locktime fails + BOOST_CHECK(!CheckFinalTx(::ChainActive().Tip(), CTransaction(tx), flags)); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), ::ChainActive().Tip()->nHeight + 2, ::ChainActive().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block @@ -472,7 +474,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 4; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); - BOOST_CHECK(!CheckFinalTx(CTransaction(tx), flags)); // Locktime fails + BOOST_CHECK(!CheckFinalTx(::ChainActive().Tip(), CTransaction(tx), flags)); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), ::ChainActive().Tip()->nHeight + 2, ::ChainActive().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later @@ -481,7 +483,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = ::ChainActive().Tip()->nHeight + 1; tx.nLockTime = 0; tx.vin[0].nSequence = 0; - BOOST_CHECK(CheckFinalTx(CTransaction(tx), flags)); // Locktime passes + BOOST_CHECK(CheckFinalTx(::ChainActive().Tip(), CTransaction(tx), flags)); // Locktime passes BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass tx.vin[0].nSequence = 1; BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index a1b41e17ed..f039f76f82 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -192,14 +192,15 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) id++, NODE_NETWORK, hSocket, addr, /* nKeyedNetGroupIn = */ 0, /* nLocalHostNonceIn = */ 0, - CAddress(), pszDest, ConnectionType::OUTBOUND_FULL_RELAY); + CAddress(), pszDest, ConnectionType::OUTBOUND_FULL_RELAY, + /* inbound_onion = */ false); BOOST_CHECK(pnode1->IsFullOutboundConn() == true); BOOST_CHECK(pnode1->IsManualConn() == false); BOOST_CHECK(pnode1->IsBlockOnlyConn() == false); BOOST_CHECK(pnode1->IsFeelerConn() == false); BOOST_CHECK(pnode1->IsAddrFetchConn() == false); BOOST_CHECK(pnode1->IsInboundConn() == false); - BOOST_CHECK(pnode1->IsInboundOnion() == false); + BOOST_CHECK(pnode1->m_inbound_onion == false); BOOST_CHECK_EQUAL(pnode1->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode2 = MakeUnique<CNode>( @@ -214,7 +215,7 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode2->IsFeelerConn() == false); BOOST_CHECK(pnode2->IsAddrFetchConn() == false); BOOST_CHECK(pnode2->IsInboundConn() == true); - BOOST_CHECK(pnode2->IsInboundOnion() == false); + BOOST_CHECK(pnode2->m_inbound_onion == false); BOOST_CHECK_EQUAL(pnode2->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode3 = MakeUnique<CNode>( @@ -229,7 +230,7 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode3->IsFeelerConn() == false); BOOST_CHECK(pnode3->IsAddrFetchConn() == false); BOOST_CHECK(pnode3->IsInboundConn() == false); - BOOST_CHECK(pnode3->IsInboundOnion() == false); + BOOST_CHECK(pnode3->m_inbound_onion == false); BOOST_CHECK_EQUAL(pnode3->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode4 = MakeUnique<CNode>( @@ -244,7 +245,7 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode4->IsFeelerConn() == false); BOOST_CHECK(pnode4->IsAddrFetchConn() == false); BOOST_CHECK(pnode4->IsInboundConn() == true); - BOOST_CHECK(pnode4->IsInboundOnion() == true); + BOOST_CHECK(pnode4->m_inbound_onion == true); BOOST_CHECK_EQUAL(pnode4->ConnectedThroughNetwork(), Network::NET_ONION); } @@ -321,6 +322,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsTor()); + BOOST_CHECK(!addr.IsI2P()); BOOST_CHECK(!addr.IsBindAny()); BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); @@ -331,6 +333,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsTor()); + BOOST_CHECK(!addr.IsI2P()); BOOST_CHECK(!addr.IsBindAny()); BOOST_CHECK(!addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToString(), torv3_addr); @@ -351,6 +354,35 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) // TOR, invalid base32 BOOST_CHECK(!addr.SetSpecial(std::string{"mf*g zak.onion"})); + // I2P + const char* i2p_addr = "UDHDrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.I2P"; + BOOST_REQUIRE(addr.SetSpecial(i2p_addr)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsI2P()); + + BOOST_CHECK(!addr.IsTor()); + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK(!addr.IsAddrV1Compatible()); + BOOST_CHECK_EQUAL(addr.ToString(), ToLower(i2p_addr)); + + // I2P, correct length, but decodes to less than the expected number of bytes. + BOOST_CHECK(!addr.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jn=.b32.i2p")); + + // I2P, extra unnecessary padding + BOOST_CHECK(!addr.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna=.b32.i2p")); + + // I2P, malicious + BOOST_CHECK(!addr.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v\0wtf.b32.i2p"s)); + + // I2P, valid but unsupported (56 Base32 characters) + // See "Encrypted LS with Base 32 Addresses" in + // https://geti2p.net/spec/encryptedleaseset.txt + BOOST_CHECK( + !addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscsad.b32.i2p")); + + // I2P, invalid base32 + BOOST_CHECK(!addr.SetSpecial(std::string{"tp*szydbh4dp.b32.i2p"})); + // Internal addr.SetInternal("esffpp"); BOOST_REQUIRE(!addr.IsValid()); // "internal" is considered invalid @@ -679,7 +711,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) in_addr ipv4AddrPeer; ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); - std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, INVALID_SOCKET, addr, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY); + std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress{}, /* pszDest */ std::string{}, ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false); pnode->fSuccessfullyConnected.store(true); // the peer claims to be reaching us via IPv6 @@ -690,7 +722,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) pnode->SetAddrLocal(addrLocal); // before patch, this causes undefined behavior detectable with clang's -fsanitize=memory - AdvertiseLocal(&*pnode); + GetLocalAddrForPeer(&*pnode); // suppress no-checks-run warning; if this test fails, it's by triggering a sanitizer BOOST_CHECK(1); @@ -793,7 +825,7 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_c candidates.push_back({ /* id */ id, /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* nMinPingUsecTime */ static_cast<int64_t>(random_context.randrange(100)), + /* m_min_ping_time */ static_cast<int64_t>(random_context.randrange(100)), /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), /* fRelevantServices */ random_context.randbool(), @@ -853,7 +885,7 @@ BOOST_AUTO_TEST_CASE(node_eviction_test) // from eviction. BOOST_CHECK(!IsEvicted( number_of_nodes, [](NodeEvictionCandidate& candidate) { - candidate.nMinPingUsecTime = candidate.id; + candidate.m_min_ping_time = candidate.id; }, {0, 1, 2, 3, 4, 5, 6, 7}, random_context)); @@ -900,7 +932,7 @@ BOOST_AUTO_TEST_CASE(node_eviction_test) BOOST_CHECK(!IsEvicted( number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.nKeyedNetGroup = number_of_nodes - candidate.id; // 4 protected - candidate.nMinPingUsecTime = candidate.id; // 8 protected + candidate.m_min_ping_time = candidate.id; // 8 protected candidate.nLastTXTime = number_of_nodes - candidate.id; // 4 protected candidate.nLastBlockTime = number_of_nodes - candidate.id; // 4 protected }, diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index ac4db3a5b6..66ad7bb5ea 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -226,8 +226,22 @@ BOOST_AUTO_TEST_CASE(subnet_test) // IPv4 address with IPv6 netmask or the other way around. BOOST_CHECK(!CSubNet(ResolveIP("1.1.1.1"), ResolveIP("ffff::")).IsValid()); BOOST_CHECK(!CSubNet(ResolveIP("::1"), ResolveIP("255.0.0.0")).IsValid()); - // Can't subnet TOR (or any other non-IPv4 and non-IPv6 network). - BOOST_CHECK(!CSubNet(ResolveIP("5wyqrzbvrdsumnok.onion"), ResolveIP("255.0.0.0")).IsValid()); + + // Create Non-IP subnets. + + const CNetAddr tor_addr{ + ResolveIP("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion")}; + + subnet = CSubNet(tor_addr); + BOOST_CHECK(subnet.IsValid()); + BOOST_CHECK_EQUAL(subnet.ToString(), tor_addr.ToString()); + BOOST_CHECK(subnet.Match(tor_addr)); + BOOST_CHECK( + !subnet.Match(ResolveIP("kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion"))); + BOOST_CHECK(!subnet.Match(ResolveIP("1.2.3.4"))); + + BOOST_CHECK(!CSubNet(tor_addr, 200).IsValid()); + BOOST_CHECK(!CSubNet(tor_addr, ResolveIP("255.0.0.0")).IsValid()); subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); @@ -442,8 +456,7 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com"s, ret)); BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com\0"s, ret)); - // We only do subnetting for IPv4 and IPv6 - BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion"s, ret)); + BOOST_CHECK(LookupSubNet("5wyqrzbvrdsumnok.onion"s, ret)); BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0"s, ret)); BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0example.com"s, ret)); BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0example.com\0"s, ret)); diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 740b2c72db..3e4b963fe3 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -5,6 +5,7 @@ #include <compat/sanity.h> #include <key.h> #include <test/util/setup_common.h> +#include <util/time.h> #include <boost/test/unit_test.hpp> @@ -15,6 +16,7 @@ BOOST_AUTO_TEST_CASE(basic_sanity) BOOST_CHECK_MESSAGE(glibc_sanity_test() == true, "libc sanity test"); BOOST_CHECK_MESSAGE(glibcxx_sanity_test() == true, "stdlib sanity test"); BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test"); + BOOST_CHECK_MESSAGE(ChronoSanityCheck() == true, "chrono epoch test"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 2e5a7549b7..d57c000b92 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -7,9 +7,11 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> +#include <functional> #include <mutex> +#include <thread> +#include <vector> BOOST_AUTO_TEST_SUITE(scheduler_tests) @@ -68,16 +70,16 @@ BOOST_AUTO_TEST_CASE(manythreads) BOOST_CHECK(last > now); // As soon as these are created they will start running and servicing the queue - boost::thread_group microThreads; + std::vector<std::thread> microThreads; for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); UninterruptibleSleep(std::chrono::microseconds{600}); now = std::chrono::system_clock::now(); // More threads and more tasks: for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); for (int i = 0; i < 100; i++) { std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); @@ -90,7 +92,10 @@ BOOST_AUTO_TEST_CASE(manythreads) // Drain the task queue then exit threads microTasks.StopWhenDrained(); - microThreads.join_all(); // ... wait until all the threads are done + // wait until all the threads are done + for (auto& thread: microThreads) { + if (thread.joinable()) thread.join(); + } int counterSum = 0; for (int i = 0; i < 10; i++) { @@ -130,9 +135,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // if the queues only permit execution of one task at once then // the extra threads should effectively be doing nothing // if they don't we'll get out of order behaviour - boost::thread_group threads; + std::vector<std::thread> threads; for (int i = 0; i < 5; ++i) { - threads.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + threads.emplace_back(std::bind(&CScheduler::serviceQueue, &scheduler)); } // these are not atomic, if SinglethreadedSchedulerClient prevents @@ -156,7 +161,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // finish up scheduler.StopWhenDrained(); - threads.join_all(); + for (auto& thread: threads) { + if (thread.joinable()) thread.join(); + } BOOST_CHECK_EQUAL(counter1, 100); BOOST_CHECK_EQUAL(counter2, 100); diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index fb84b7f2ac..4dc0dd5f51 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -107,6 +107,22 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(scriptHash)); + // TxoutType::WITNESS_V1_TAPROOT + s.clear(); + s << OP_1 << ToByteVector(uint256::ZERO); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V1_TAPROOT); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{1}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ZERO)); + + // TxoutType::WITNESS_UNKNOWN + s.clear(); + s << OP_16 << ToByteVector(uint256::ONE); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_UNKNOWN); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{16}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ONE)); + // TxoutType::NONSTANDARD s.clear(); s << OP_9 << OP_ADD << OP_11 << OP_EQUAL; @@ -183,23 +199,23 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) s.clear(); s << ToByteVector(pubkey) << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<PKHash>(&address) && - *boost::get<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get_if<PKHash>(&address) && + *std::get_if<PKHash>(&address) == PKHash(pubkey)); // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<PKHash>(&address) && - *boost::get<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get_if<PKHash>(&address) && + *std::get_if<PKHash>(&address) == PKHash(pubkey)); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<ScriptHash>(&address) && - *boost::get<ScriptHash>(&address) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get_if<ScriptHash>(&address) && + *std::get_if<ScriptHash>(&address) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -217,7 +233,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(ExtractDestination(s, address)); WitnessV0KeyHash keyhash; CHash160().Write(pubkey).Finalize(keyhash); - BOOST_CHECK(boost::get<WitnessV0KeyHash>(&address) && *boost::get<WitnessV0KeyHash>(&address) == keyhash); + BOOST_CHECK(std::get_if<WitnessV0KeyHash>(&address) && *std::get_if<WitnessV0KeyHash>(&address) == keyhash); // TxoutType::WITNESS_V0_SCRIPTHASH s.clear(); @@ -225,7 +241,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CSHA256().Write(redeemScript.data(), redeemScript.size()).Finalize(scripthash.begin()); s << OP_0 << ToByteVector(scripthash); BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<WitnessV0ScriptHash>(&address) && *boost::get<WitnessV0ScriptHash>(&address) == scripthash); + BOOST_CHECK(std::get_if<WitnessV0ScriptHash>(&address) && *std::get_if<WitnessV0ScriptHash>(&address) == scripthash); // TxoutType::WITNESS_UNKNOWN with unknown version s.clear(); @@ -235,7 +251,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) unk.length = 33; unk.version = 1; std::copy(pubkey.begin(), pubkey.end(), unk.program); - BOOST_CHECK(boost::get<WitnessUnknown>(&address) && *boost::get<WitnessUnknown>(&address) == unk); + BOOST_CHECK(std::get_if<WitnessUnknown>(&address) && *std::get_if<WitnessUnknown>(&address) == unk); } BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) @@ -259,8 +275,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); // TxoutType::PUBKEYHASH s.clear(); @@ -269,8 +285,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script @@ -280,8 +296,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<ScriptHash>(&addresses[0]) && - *boost::get<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get_if<ScriptHash>(&addresses[0]) && + *std::get_if<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -293,10 +309,10 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); BOOST_CHECK_EQUAL(addresses.size(), 2U); BOOST_CHECK_EQUAL(nRequired, 2); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - BOOST_CHECK(boost::get<PKHash>(&addresses[1]) && - *boost::get<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[1]) && + *std::get_if<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); // TxoutType::NULL_DATA s.clear(); diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 281018be9f..746d4d3c6b 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -164,9 +164,9 @@ static void RunOperators(const int64_t& num1, const int64_t& num2) BOOST_AUTO_TEST_CASE(creation) { - for(size_t i = 0; i < sizeof(values) / sizeof(values[0]); ++i) + for(size_t i = 0; i < std::size(values); ++i) { - for(size_t j = 0; j < sizeof(offsets) / sizeof(offsets[0]); ++j) + for(size_t j = 0; j < std::size(offsets); ++j) { RunCreate(values[i]); RunCreate(values[i] + offsets[j]); @@ -177,9 +177,9 @@ BOOST_AUTO_TEST_CASE(creation) BOOST_AUTO_TEST_CASE(operators) { - for(size_t i = 0; i < sizeof(values) / sizeof(values[0]); ++i) + for(size_t i = 0; i < std::size(values); ++i) { - for(size_t j = 0; j < sizeof(offsets) / sizeof(offsets[0]); ++j) + for(size_t j = 0; j < std::size(offsets); ++j) { RunOperators(values[i], values[i]); RunOperators(values[i], -values[i]); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index f625b67c2a..f77cda7ba2 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.insert(ss.end(), c); BOOST_CHECK_EQUAL(ss.size(), 6U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); BOOST_CHECK_EQUAL(ss[5], c); ss.insert(ss.begin()+2, c); @@ -334,19 +334,14 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.erase(ss.begin()+ss.size()-1); BOOST_CHECK_EQUAL(ss.size(), 5U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); ss.erase(ss.begin()+1); BOOST_CHECK_EQUAL(ss.size(), 4U); BOOST_CHECK_EQUAL(ss[0], 0); BOOST_CHECK_EQUAL(ss[1], 1); BOOST_CHECK_EQUAL(ss[2], 2); - BOOST_CHECK_EQUAL(ss[3], (char)0xff); - - // Make sure GetAndClear does the right thing: - CSerializeData d; - ss.GetAndClear(d); - BOOST_CHECK_EQUAL(ss.size(), 0U); + BOOST_CHECK_EQUAL(ss[3], 0xff); } BOOST_AUTO_TEST_CASE(class_methods) diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index bc862de78a..2eb980e8cd 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -88,7 +88,7 @@ void static RandomScript(CScript &script) { script = CScript(); int ops = (InsecureRandRange(10)); for (int i=0; i<ops; i++) - script << oplist[InsecureRandRange(sizeof(oplist)/sizeof(oplist[0]))]; + script << oplist[InsecureRandRange(std::size(oplist))]; } void static RandomTransaction(CMutableTransaction &tx, bool fSingle) { diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp new file mode 100644 index 0000000000..ed9780dfb5 --- /dev/null +++ b/src/test/sock_tests.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2021-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <compat.h> +#include <test/util/setup_common.h> +#include <util/sock.h> +#include <util/system.h> + +#include <boost/test/unit_test.hpp> + +#include <thread> + +using namespace std::chrono_literals; + +BOOST_FIXTURE_TEST_SUITE(sock_tests, BasicTestingSetup) + +static bool SocketIsClosed(const SOCKET& s) +{ + // Notice that if another thread is running and creates its own socket after `s` has been + // closed, it may be assigned the same file descriptor number. In this case, our test will + // wrongly pretend that the socket is not closed. + int type; + socklen_t len = sizeof(type); + return getsockopt(s, SOL_SOCKET, SO_TYPE, (sockopt_arg_type)&type, &len) == SOCKET_ERROR; +} + +static SOCKET CreateSocket() +{ + const SOCKET s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + BOOST_REQUIRE(s != static_cast<SOCKET>(SOCKET_ERROR)); + return s; +} + +BOOST_AUTO_TEST_CASE(constructor_and_destructor) +{ + const SOCKET s = CreateSocket(); + Sock* sock = new Sock(s); + BOOST_CHECK_EQUAL(sock->Get(), s); + BOOST_CHECK(!SocketIsClosed(s)); + delete sock; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(move_constructor) +{ + const SOCKET s = CreateSocket(); + Sock* sock1 = new Sock(s); + Sock* sock2 = new Sock(std::move(*sock1)); + delete sock1; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_CHECK_EQUAL(sock2->Get(), s); + delete sock2; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(move_assignment) +{ + const SOCKET s = CreateSocket(); + Sock* sock1 = new Sock(s); + Sock* sock2 = new Sock(); + *sock2 = std::move(*sock1); + delete sock1; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_CHECK_EQUAL(sock2->Get(), s); + delete sock2; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(release) +{ + SOCKET s = CreateSocket(); + Sock* sock = new Sock(s); + BOOST_CHECK_EQUAL(sock->Release(), s); + delete sock; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_REQUIRE(CloseSocket(s)); +} + +BOOST_AUTO_TEST_CASE(reset) +{ + const SOCKET s = CreateSocket(); + Sock sock(s); + sock.Reset(); + BOOST_CHECK(SocketIsClosed(s)); +} + +#ifndef WIN32 // Windows does not have socketpair(2). + +static void CreateSocketPair(int s[2]) +{ + BOOST_REQUIRE_EQUAL(socketpair(AF_UNIX, SOCK_STREAM, 0, s), 0); +} + +static void SendAndRecvMessage(const Sock& sender, const Sock& receiver) +{ + const char* msg = "abcd"; + constexpr ssize_t msg_len = 4; + char recv_buf[10]; + + BOOST_CHECK_EQUAL(sender.Send(msg, msg_len, 0), msg_len); + BOOST_CHECK_EQUAL(receiver.Recv(recv_buf, sizeof(recv_buf), 0), msg_len); + BOOST_CHECK_EQUAL(strncmp(msg, recv_buf, msg_len), 0); +} + +BOOST_AUTO_TEST_CASE(send_and_receive) +{ + int s[2]; + CreateSocketPair(s); + + Sock* sock0 = new Sock(s[0]); + Sock* sock1 = new Sock(s[1]); + + SendAndRecvMessage(*sock0, *sock1); + + Sock* sock0moved = new Sock(std::move(*sock0)); + Sock* sock1moved = new Sock(); + *sock1moved = std::move(*sock1); + + delete sock0; + delete sock1; + + SendAndRecvMessage(*sock1moved, *sock0moved); + + delete sock0moved; + delete sock1moved; + + BOOST_CHECK(SocketIsClosed(s[0])); + BOOST_CHECK(SocketIsClosed(s[1])); +} + +BOOST_AUTO_TEST_CASE(wait) +{ + int s[2]; + CreateSocketPair(s); + + Sock sock0(s[0]); + Sock sock1(s[1]); + + std::thread waiter([&sock0]() { sock0.Wait(24h, Sock::RECV); }); + + BOOST_REQUIRE_EQUAL(sock1.Send("a", 1, 0), 1); + + waiter.join(); +} + +#endif /* WIN32 */ + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 8f5ee1e8e2..3079c9ff29 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { - std::vector<char> in; + std::vector<uint8_t> in; std::vector<char> expected_xor; std::vector<unsigned char> key; CDataStream ds(in, 0, 0); diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 71275f69d9..3e4d1dac9e 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -6,7 +6,6 @@ #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/mutex.hpp> #include <mutex> @@ -110,11 +109,6 @@ BOOST_AUTO_TEST_CASE(double_lock_mutex) TestDoubleLock<Mutex>(true /* should throw */); } -BOOST_AUTO_TEST_CASE(double_lock_boost_mutex) -{ - TestDoubleLock<boost::mutex>(true /* should throw */); -} - BOOST_AUTO_TEST_CASE(double_lock_recursive_mutex) { TestDoubleLock<RecursiveMutex>(false /* should not throw */); diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index ce555f7299..940145b84f 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -6,22 +6,22 @@ #include <util/system.h> #include <univalue.h> -#ifdef HAVE_BOOST_PROCESS +#ifdef ENABLE_EXTERNAL_SIGNER #include <boost/process.hpp> -#endif // HAVE_BOOST_PROCESS +#endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(system_tests, BasicTestingSetup) -// At least one test is required (in case HAVE_BOOST_PROCESS is not defined). +// At least one test is required (in case ENABLE_EXTERNAL_SIGNER is not defined). // Workaround for https://github.com/bitcoin/bitcoin/issues/19128 BOOST_AUTO_TEST_CASE(dummy) { BOOST_CHECK(true); } -#ifdef HAVE_BOOST_PROCESS +#ifdef ENABLE_EXTERNAL_SIGNER bool checkMessage(const std::runtime_error& ex) { @@ -90,6 +90,6 @@ BOOST_AUTO_TEST_CASE(run_command) } #endif } -#endif // HAVE_BOOST_PROCESS +#endif // ENABLE_EXTERNAL_SIGNER BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 5b35ed6976..8f1d99b199 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -22,8 +22,10 @@ #include <streams.h> #include <test/util/transaction_utils.h> #include <util/strencodings.h> +#include <util/string.h> #include <validation.h> +#include <functional> #include <map> #include <string> @@ -39,7 +41,6 @@ typedef std::vector<unsigned char> valtype; extern UniValue read_json(const std::string& jsondata); static std::map<std::string, unsigned int> mapFlagNames = { - {std::string("NONE"), (unsigned int)SCRIPT_VERIFY_NONE}, {std::string("P2SH"), (unsigned int)SCRIPT_VERIFY_P2SH}, {std::string("STRICTENC"), (unsigned int)SCRIPT_VERIFY_STRICTENC}, {std::string("DERSIG"), (unsigned int)SCRIPT_VERIFY_DERSIG}, @@ -62,9 +63,7 @@ static std::map<std::string, unsigned int> mapFlagNames = { unsigned int ParseScriptFlags(std::string strFlags) { - if (strFlags.empty()) { - return 0; - } + if (strFlags.empty() || strFlags == "NONE") return 0; unsigned int flags = 0; std::vector<std::string> words; boost::algorithm::split(words, strFlags, boost::algorithm::is_any_of(",")); @@ -95,20 +94,84 @@ std::string FormatScriptFlags(unsigned int flags) return ret.substr(0, ret.size() - 1); } +/* +* Check that the input scripts of a transaction are valid/invalid as expected. +*/ +bool CheckTxScripts(const CTransaction& tx, const std::map<COutPoint, CScript>& map_prevout_scriptPubKeys, + const std::map<COutPoint, int64_t>& map_prevout_values, unsigned int flags, + const PrecomputedTransactionData& txdata, const std::string& strTest, bool expect_valid) +{ + bool tx_valid = true; + ScriptError err = expect_valid ? SCRIPT_ERR_UNKNOWN_ERROR : SCRIPT_ERR_OK; + for (unsigned int i = 0; i < tx.vin.size() && tx_valid; ++i) { + const CTxIn input = tx.vin[i]; + const CAmount amount = map_prevout_values.count(input.prevout) ? map_prevout_values.at(input.prevout) : 0; + try { + tx_valid = VerifyScript(input.scriptSig, map_prevout_scriptPubKeys.at(input.prevout), + &input.scriptWitness, flags, TransactionSignatureChecker(&tx, i, amount, txdata), &err); + } catch (...) { + BOOST_ERROR("Bad test: " << strTest); + return true; // The test format is bad and an error is thrown. Return true to silence further error. + } + if (expect_valid) { + BOOST_CHECK_MESSAGE(tx_valid, strTest); + BOOST_CHECK_MESSAGE((err == SCRIPT_ERR_OK), ScriptErrorString(err)); + err = SCRIPT_ERR_UNKNOWN_ERROR; + } + } + if (!expect_valid) { + BOOST_CHECK_MESSAGE(!tx_valid, strTest); + BOOST_CHECK_MESSAGE((err != SCRIPT_ERR_OK), ScriptErrorString(err)); + } + return (tx_valid == expect_valid); +} + +/* + * Trim or fill flags to make the combination valid: + * WITNESS must be used with P2SH + * CLEANSTACK must be used WITNESS and P2SH + */ + +unsigned int TrimFlags(unsigned int flags) +{ + // WITNESS requires P2SH + if (!(flags & SCRIPT_VERIFY_P2SH)) flags &= ~(unsigned int)SCRIPT_VERIFY_WITNESS; + + // CLEANSTACK requires WITNESS (and transitively CLEANSTACK requires P2SH) + if (!(flags & SCRIPT_VERIFY_WITNESS)) flags &= ~(unsigned int)SCRIPT_VERIFY_CLEANSTACK; + return flags; +} + +unsigned int FillFlags(unsigned int flags) +{ + // CLEANSTACK implies WITNESS + if (flags & SCRIPT_VERIFY_CLEANSTACK) flags |= SCRIPT_VERIFY_WITNESS; + + // WITNESS implies P2SH (and transitively CLEANSTACK implies P2SH) + if (flags & SCRIPT_VERIFY_WITNESS) flags |= SCRIPT_VERIFY_P2SH; + return flags; +} + +// Return valid flags that are all except one flag for each flag +std::vector<unsigned int> ExcludeIndividualFlags(unsigned int flags) +{ + std::vector<unsigned int> flags_combos; + for (unsigned int i = 0; i < mapFlagNames.size(); ++i) { + const unsigned int flags_excluding_i = TrimFlags(flags & ~(1U << i)); + if (flags != flags_excluding_i && std::find(flags_combos.begin(), flags_combos.end(), flags_excluding_i) != flags_combos.end()) { + flags_combos.push_back(flags_excluding_i); + } + } + return flags_combos; +} + BOOST_FIXTURE_TEST_SUITE(transaction_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(tx_valid) { // Read tests from test/data/tx_valid.json - // Format is an array of arrays - // Inner arrays are either [ "comment" ] - // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, verifyFlags - // ... where all scripts are stringified scripts. - // - // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" UniValue tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid))); - ScriptError err; for (unsigned int idx = 0; idx < tests.size(); idx++) { UniValue test = tests[idx]; std::string strTest = test.write(); @@ -158,24 +221,36 @@ BOOST_AUTO_TEST_CASE(tx_valid) BOOST_CHECK(state.IsValid()); PrecomputedTransactionData txdata(tx); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if (!mapprevOutScriptPubKeys.count(tx.vin[i].prevout)) - { - BOOST_ERROR("Bad test: " << strTest); - break; + unsigned int verify_flags = ParseScriptFlags(test[2].get_str()); + + // Check that the test gives a valid combination of flags (otherwise VerifyScript will throw). Don't edit the flags. + if (~verify_flags != FillFlags(~verify_flags)) { + BOOST_ERROR("Bad test flags: " << strTest); + } + + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~verify_flags, txdata, strTest, /* expect_valid */ true)) { + BOOST_ERROR("Tx unexpectedly failed: " << strTest); + } + + // Backwards compatibility of script verification flags: Removing any flag(s) should not invalidate a valid transaction + for (size_t i = 0; i < mapFlagNames.size(); ++i) { + // Removing individual flags + unsigned int flags = TrimFlags(~(verify_flags | (1U << i))); + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ true)) { + BOOST_ERROR("Tx unexpectedly failed with flag " << ToString(i) << " unset: " << strTest); } + // Removing random combinations of flags + flags = TrimFlags(~(verify_flags | (unsigned int)InsecureRandBits(mapFlagNames.size()))); + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ true)) { + BOOST_ERROR("Tx unexpectedly failed with random flags " << ToString(flags) << ": " << strTest); + } + } - CAmount amount = 0; - if (mapprevOutValues.count(tx.vin[i].prevout)) { - amount = mapprevOutValues[tx.vin[i].prevout]; + // Check that flags are maximal: transaction should fail if any unset flags are set. + for (auto flags_excluding_one: ExcludeIndividualFlags(verify_flags)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~flags_excluding_one, txdata, strTest, /* expect_valid */ false)) { + BOOST_ERROR("Too many flags unset: " << strTest); } - unsigned int verify_flags = ParseScriptFlags(test[2].get_str()); - const CScriptWitness *witness = &tx.vin[i].scriptWitness; - BOOST_CHECK_MESSAGE(VerifyScript(tx.vin[i].scriptSig, mapprevOutScriptPubKeys[tx.vin[i].prevout], - witness, verify_flags, TransactionSignatureChecker(&tx, i, amount, txdata), &err), - strTest); - BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); } } } @@ -184,17 +259,8 @@ BOOST_AUTO_TEST_CASE(tx_valid) BOOST_AUTO_TEST_CASE(tx_invalid) { // Read tests from test/data/tx_invalid.json - // Format is an array of arrays - // Inner arrays are either [ "comment" ] - // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, verifyFlags - // ... where all scripts are stringified scripts. - // - // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" UniValue tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid))); - // Initialize to SCRIPT_ERR_OK. The tests expect err to be changed to a - // value other than SCRIPT_ERR_OK. - ScriptError err = SCRIPT_ERR_OK; for (unsigned int idx = 0; idx < tests.size(); idx++) { UniValue test = tests[idx]; std::string strTest = test.write(); @@ -240,28 +306,39 @@ BOOST_AUTO_TEST_CASE(tx_invalid) CTransaction tx(deserialize, stream); TxValidationState state; - fValid = CheckTransaction(tx, state) && state.IsValid(); + if (!CheckTransaction(tx, state) || state.IsInvalid()) { + BOOST_CHECK_MESSAGE(test[2].get_str() == "BADTX", strTest); + continue; + } PrecomputedTransactionData txdata(tx); - for (unsigned int i = 0; i < tx.vin.size() && fValid; i++) - { - if (!mapprevOutScriptPubKeys.count(tx.vin[i].prevout)) - { - BOOST_ERROR("Bad test: " << strTest); - break; + unsigned int verify_flags = ParseScriptFlags(test[2].get_str()); + + // Not using FillFlags() in the main test, in order to detect invalid verifyFlags combination + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, verify_flags, txdata, strTest, /* expect_valid */ false)) { + BOOST_ERROR("Tx unexpectedly passed: " << strTest); + } + + // Backwards compatibility of script verification flags: Adding any flag(s) should not validate an invalid transaction + for (size_t i = 0; i < mapFlagNames.size(); i++) { + unsigned int flags = FillFlags(verify_flags | (1U << i)); + // Adding individual flags + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ false)) { + BOOST_ERROR("Tx unexpectedly passed with flag " << ToString(i) << " set: " << strTest); + } + // Adding random combinations of flags + flags = FillFlags(verify_flags | (unsigned int)InsecureRandBits(mapFlagNames.size())); + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ false)) { + BOOST_ERROR("Tx unexpectedly passed with random flags " << ToString(flags) << ": " << strTest); } + } - unsigned int verify_flags = ParseScriptFlags(test[2].get_str()); - CAmount amount = 0; - if (mapprevOutValues.count(tx.vin[i].prevout)) { - amount = mapprevOutValues[tx.vin[i].prevout]; + // Check that flags are minimal: transaction should succeed if any set flags are unset. + for (auto flags_excluding_one: ExcludeIndividualFlags(verify_flags)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags_excluding_one, txdata, strTest, /* expect_valid */ true)) { + BOOST_ERROR("Too many flags set: " << strTest); } - const CScriptWitness *witness = &tx.vin[i].scriptWitness; - fValid = VerifyScript(tx.vin[i].scriptSig, mapprevOutScriptPubKeys[tx.vin[i].prevout], - witness, verify_flags, TransactionSignatureChecker(&tx, i, amount, txdata), &err); } - BOOST_CHECK_MESSAGE(!fValid, strTest); - BOOST_CHECK_MESSAGE(err != SCRIPT_ERR_OK, ScriptErrorString(err)); } } } @@ -428,12 +505,10 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); - boost::thread_group threadGroup; CCheckQueue<CScriptCheck> scriptcheckqueue(128); CCheckQueueControl<CScriptCheck> control(&scriptcheckqueue); - for (int i=0; i<20; i++) - threadGroup.create_thread(std::bind(&CCheckQueue<CScriptCheck>::Thread, std::ref(scriptcheckqueue))); + scriptcheckqueue.StartWorkerThreads(20); std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { @@ -455,9 +530,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) bool controlCheck = control.Wait(); assert(controlCheck); - - threadGroup.interrupt_all(); - threadGroup.join_all(); + scriptcheckqueue.StopWorkerThreads(); } SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutableTransaction& input2, const CTransactionRef tx) diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 7e6246d68f..8d14071297 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -30,25 +30,21 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK(CTransaction(coinbaseTx).IsCoinBase()); - TxValidationState state; - LOCK(cs_main); unsigned int initialPoolSize = m_node.mempool->size(); + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *m_node.mempool, MakeTransactionRef(coinbaseTx), + true /* bypass_limits */); - BOOST_CHECK_EQUAL( - false, - AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(coinbaseTx), - nullptr /* plTxnReplaced */, - true /* bypass_limits */)); + BOOST_CHECK(result.m_result_type == MempoolAcceptResult::ResultType::INVALID); // Check that the transaction hasn't been added to mempool. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); // Check that the validation state reflects the unsuccessful attempt. - BOOST_CHECK(state.IsInvalid()); - BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); - BOOST_CHECK(state.GetResult() == TxValidationResult::TX_CONSENSUS); + BOOST_CHECK(result.m_state.IsInvalid()); + BOOST_CHECK_EQUAL(result.m_state.GetRejectReason(), "coinbase"); + BOOST_CHECK(result.m_state.GetResult() == TxValidationResult::TX_CONSENSUS); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index bed2ba3608..3244b58082 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -13,7 +13,10 @@ #include <boost/test/unit_test.hpp> -bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); +bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, + const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, + bool cacheFullScriptStore, PrecomputedTransactionData& txdata, + std::vector<CScriptCheck>* pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main); BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) @@ -28,9 +31,9 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) const auto ToMemPool = [this](const CMutableTransaction& tx) { LOCK(cs_main); - TxValidationState state; - return AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(tx), - nullptr /* plTxnReplaced */, true /* bypass_limits */); + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *m_node.mempool, MakeTransactionRef(tx), + true /* bypass_limits */); + return result.m_result_type == MempoolAcceptResult::ResultType::VALID; }; // Create a double-spend of mature coinbase txn: diff --git a/src/test/util/script.h b/src/test/util/script.h new file mode 100644 index 0000000000..abd14c2067 --- /dev/null +++ b/src/test/util/script.h @@ -0,0 +1,21 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_SCRIPT_H +#define BITCOIN_TEST_UTIL_SCRIPT_H + +#include <crypto/sha256.h> +#include <script/script.h> + +static const std::vector<uint8_t> WITNESS_STACK_ELEM_OP_TRUE{uint8_t{OP_TRUE}}; +static const CScript P2WSH_OP_TRUE{ + CScript{} + << OP_0 + << ToByteVector([] { + uint256 hash; + CSHA256().Write(WITNESS_STACK_ELEM_OP_TRUE.data(), WITNESS_STACK_ELEM_OP_TRUE.size()).Finalize(hash.begin()); + return hash; + }())}; + +#endif // BITCOIN_TEST_UTIL_SCRIPT_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index e167fc98fd..1ffe435531 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -77,6 +77,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve { "dummy", "-printtoconsole=0", + "-logsourcelocations", "-logtimemicros", "-logthreadnames", "-debug", @@ -131,7 +132,7 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. m_node.scheduler = MakeUnique<CScheduler>(); - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); + m_node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -143,17 +144,14 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; - for (int i = 0; i < script_check_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_check_threads); g_parallel_script_checks = true; } ChainTestingSetup::~ChainTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); m_node.connman.reset(); @@ -186,15 +184,15 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peerman = std::make_unique<PeerManager>(chainparams, *m_node.connman, m_node.banman.get(), - *m_node.scheduler, *m_node.chainman, *m_node.mempool, - false); + m_node.peerman = PeerManager::make(chainparams, *m_node.connman, m_node.banman.get(), + *m_node.scheduler, *m_node.chainman, *m_node.mempool, + false); { CConnman::Options options; options.m_msgproc = m_node.peerman.get(); @@ -202,14 +200,43 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup() +TestChain100Setup::TestChain100Setup(bool deterministic) { + m_deterministic = deterministic; + + if (m_deterministic) { + SetMockTime(1598887952); + constexpr std::array<unsigned char, 32> vchKey = { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 + } + }; + coinbaseKey.Set(vchKey.begin(), vchKey.end(), false); + } else { + coinbaseKey.MakeNewKey(true); + } + // Generate a 100-block chain: - coinbaseKey.MakeNewKey(true); + this->mineBlocks(COINBASE_MATURITY); + + if (m_deterministic) { + LOCK(::cs_main); + assert( + m_node.chainman->ActiveChain().Tip()->GetBlockHash().ToString() == + "49c95db1e470fed04496d801c9d8fbb78155d2c7f855232c918823d2c17d0cf6"); + } +} + +void TestChain100Setup::mineBlocks(int num_blocks) +{ CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - for (int i = 0; i < COINBASE_MATURITY; i++) { + for (int i = 0; i < num_blocks; i++) + { std::vector<CMutableTransaction> noTxns; CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); + if (m_deterministic) { + SetMockTime(GetTime() + 1); + } m_coinbase_txns.push_back(b.vtx[0]); } } @@ -234,9 +261,61 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa return block; } + +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, + int input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount) +{ + // Transaction we will submit to the mempool + CMutableTransaction mempool_txn; + + // Create an input + COutPoint outpoint_to_spend(input_transaction->GetHash(), input_vout); + CTxIn input(outpoint_to_spend); + mempool_txn.vin.push_back(input); + + // Create an output + CTxOut output(output_amount, output_destination); + mempool_txn.vout.push_back(output); + + // Sign the transaction + // - Add the signing key to a keystore + FillableSigningProvider keystore; + keystore.AddKey(input_signing_key); + // - Populate a CoinsViewCache with the unspent output + CCoinsView coins_view; + CCoinsViewCache coins_cache(&coins_view); + AddCoins(coins_cache, *input_transaction.get(), input_height); + // - Use GetCoin to properly populate utxo_to_spend, + Coin utxo_to_spend; + assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); + // - Then add it to a map to pass in to SignTransaction + std::map<COutPoint, Coin> input_coins; + input_coins.insert({outpoint_to_spend, utxo_to_spend}); + // - Default signature hashing type + int nHashType = SIGHASH_ALL; + std::map<int, std::string> input_errors; + assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + + // Add transaction to the mempool + { + LOCK(cs_main); + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *m_node.mempool.get(), MakeTransactionRef(mempool_txn), /* bypass_limits */ false); + assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); + } + + return mempool_txn; +} + TestChain100Setup::~TestChain100Setup() { gArgs.ForceSetArg("-segwitheight", "0"); + if (m_deterministic) { + SetMockTime(0); + } } CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index dff8cf825e..33f24e7c44 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -17,8 +17,7 @@ #include <util/string.h> #include <type_traits> - -#include <boost/thread/thread.hpp> +#include <vector> /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -79,7 +78,6 @@ struct BasicTestingSetup { explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); -private: const fs::path m_path_root; }; @@ -88,7 +86,6 @@ private: * initialization behaviour. */ struct ChainTestingSetup : public BasicTestingSetup { - boost::thread_group threadGroup; explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); @@ -114,7 +111,7 @@ class CScript; * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ struct TestChain100Setup : public RegTestingSetup { - TestChain100Setup(); + TestChain100Setup(bool deterministic = false); /** * Create a new block with just given transactions, coinbase paying to @@ -123,12 +120,38 @@ struct TestChain100Setup : public RegTestingSetup { CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); + //! Mine a series of new blocks on the active chain. + void mineBlocks(int num_blocks); + + /** + * Create a transaction and submit to the mempool. + * + * @param input_transaction The transaction to spend + * @param input_vout The vout to spend from the input_transaction + * @param input_height The height of the block that included the input_transaction + * @param input_signing_key The key to spend the input_transaction + * @param output_destination Where to send the output + * @param output_amount How much to send + */ + CMutableTransaction CreateValidMempoolTransaction(CTransactionRef input_transaction, + int input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount = CAmount(1 * COIN)); + ~TestChain100Setup(); + bool m_deterministic; std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions CKey coinbaseKey; // private/public key needed to spend coinbase transactions }; + +struct TestChain100DeterministicSetup : public TestChain100Setup { + TestChain100DeterministicSetup() : TestChain100Setup(true) { } +}; + class CTxMemPoolEntry; struct TestMemPoolEntryHelper diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 4133f2623b..845854bd4b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -13,6 +13,7 @@ #include <test/util/setup_common.h> #include <test/util/str.h> #include <uint256.h> +#include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/spanparsing.h> @@ -1816,7 +1817,7 @@ BOOST_AUTO_TEST_CASE(test_DirIsWritable) BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), true); // Should not be able to write to a non-existent dir. - tmpdirname = tmpdirname / fs::unique_path(); + tmpdirname = GetUniquePath(tmpdirname); BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), false); fs::create_directory(tmpdirname); @@ -2200,4 +2201,17 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_CHECK_NE(message_hash1, signature_hash); } +BOOST_AUTO_TEST_CASE(remove_prefix) +{ + BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); + BOOST_CHECK_EQUAL(RemovePrefix("foo", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); + BOOST_CHECK_EQUAL(RemovePrefix("foo", "f"), "oo"); + BOOST_CHECK_EQUAL(RemovePrefix("foo", ""), "foo"); + BOOST_CHECK_EQUAL(RemovePrefix("fo", "foo"), "fo"); + BOOST_CHECK_EQUAL(RemovePrefix("f", "foo"), "f"); + BOOST_CHECK_EQUAL(RemovePrefix("", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefix("", ""), ""); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index ea17cb50f1..0c87c4d360 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -11,6 +11,7 @@ #include <pow.h> #include <random.h> #include <script/standard.h> +#include <test/util/script.h> #include <test/util/setup_common.h> #include <util/time.h> #include <validation.h> @@ -18,8 +19,6 @@ #include <thread> -static const std::vector<unsigned char> V_OP_TRUE{OP_TRUE}; - namespace validation_block_tests { struct MinerTestingSetup : public RegTestingSetup { std::shared_ptr<CBlock> Block(const uint256& prev_hash); @@ -64,27 +63,17 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) static int i = 0; static uint64_t time = Params().GenesisBlock().nTime; - CScript pubKey; - pubKey << i++ << OP_TRUE; - - auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(pubKey); + auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(CScript{} << i++ << OP_TRUE); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; - pubKey.clear(); - { - WitnessV0ScriptHash witness_program; - CSHA256().Write(&V_OP_TRUE[0], V_OP_TRUE.size()).Finalize(witness_program.begin()); - pubKey << OP_0 << ToByteVector(witness_program); - } - // Make the coinbase transaction with two outputs: // One zero-value one that has a unique pubkey to make sure that blocks at the same height can have a different hash // Another one that has the coinbase reward in a P2WSH with OP_TRUE as witness program to make it easy to spend CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.vout.resize(2); - txCoinbase.vout[1].scriptPubKey = pubKey; + txCoinbase.vout[1].scriptPubKey = P2WSH_OP_TRUE; txCoinbase.vout[1].nValue = txCoinbase.vout[0].nValue; txCoinbase.vout[0].nValue = 0; txCoinbase.vin[0].scriptWitness.SetNull(); @@ -95,8 +84,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { - LOCK(cs_main); // For LookupBlockIndex - GenerateCoinbaseCommitment(*pblock, LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); + LOCK(cs_main); // For g_chainman.m_blockman.LookupBlockIndex + GenerateCoinbaseCommitment(*pblock, g_chainman.m_blockman.LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); @@ -254,7 +243,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) for (int num_txs = 22; num_txs > 0; --num_txs) { CMutableTransaction mtx; mtx.vin.push_back(CTxIn{COutPoint{last_mined->vtx[0]->GetHash(), 1}, CScript{}}); - mtx.vin[0].scriptWitness.stack.push_back(V_OP_TRUE); + mtx.vin[0].scriptWitness.stack.push_back(WITNESS_STACK_ELEM_OP_TRUE); mtx.vout.push_back(last_mined->vtx[0]->vout[1]); mtx.vout[0].nValue -= 1000; txs.push_back(MakeTransactionRef(mtx)); @@ -283,15 +272,9 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Add the txs to the tx pool { LOCK(cs_main); - TxValidationState state; - std::list<CTransactionRef> plTxnReplaced; for (const auto& tx : txs) { - BOOST_REQUIRE(AcceptToMemoryPool( - *m_node.mempool, - state, - tx, - &plTxnReplaced, - /* bypass_limits */ false)); + const MempoolAcceptResult result = AcceptToMemoryPool(::ChainstateActive(), *m_node.mempool, tx, false /* bypass_limits */); + BOOST_REQUIRE(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index c8a375275f..92d8cf2e7d 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 75939e0140..94d4277019 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -4,13 +4,18 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/utxo_snapshot.h> #include <random.h> +#include <rpc/blockchain.h> #include <sync.h> #include <test/util/setup_common.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> +#include <tinyformat.h> +#include <univalue.h> + #include <vector> #include <boost/test/unit_test.hpp> @@ -28,9 +33,11 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) std::vector<CChainState*> chainstates; const CChainParams& chainparams = Params(); + BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); + // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -54,10 +61,17 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& validated_cs = manager.ValidatedChainstate(); BOOST_CHECK_EQUAL(&validated_cs, &c1); + BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); + // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); + const uint256 snapshot_blockhash = GetRandHash(); + CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( + mempool, snapshot_blockhash)); chainstates.push_back(&c2); + + BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); + c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); @@ -116,7 +130,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -134,7 +148,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -155,4 +169,175 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } +auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +template<typename F = decltype(NoMalleation)> +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + +//! Test basic snapshot activation. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100DeterministicSetup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + + size_t initial_size; + size_t initial_total_coins{100}; + + // Make some initial assertions about the contents of the chainstate. + { + LOCK(::cs_main); + CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); + initial_size = ibd_coinscache.GetCacheSize(); + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(ibd_coinscache.HaveCoin(op)); + total_coins++; + } + + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + BOOST_CHECK_EQUAL(initial_size, initial_total_coins); + } + + // Snapshot should refuse to load at this height. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + BOOST_CHECK(chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull()); + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + chainman.SnapshotBlockhash().value_or(uint256())); + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + initial_size += 10; + initial_total_coins += 10; + + // Should not load malleated snapshots + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // A UTXO is missing but count is correct + metadata.m_coins_count -= 1; + + COutPoint outpoint; + Coin coin; + + auto_infile >> outpoint; + auto_infile >> coin; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is larger than coins in file + metadata.m_coins_count += 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is smaller than coins in file + metadata.m_coins_count -= 1; + })); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull()); + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + *chainman.SnapshotBlockhash()); + + // To be checked against later when we try loading a subsequent snapshot. + uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; + + // Make some assertions about the both chainstates. These checks ensure the + // legacy chainstate hasn't changed and that the newly created chainstate + // reflects the expected content. + { + LOCK(::cs_main); + int chains_tested{0}; + + for (CChainState* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + + // Both caches will be empty initially. + BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(coinscache.HaveCoin(op)); + total_coins++; + } + + BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + chains_tested++; + } + + BOOST_CHECK_EQUAL(chains_tested, 2); + } + + // Mine some new blocks on top of the activated snapshot chainstate. + constexpr size_t new_coins{100}; + mineBlocks(new_coins); // Defined in TestChain100Setup. + + { + LOCK(::cs_main); + size_t coins_in_active{0}; + size_t coins_in_ibd{0}; + size_t coins_missing_ibd{0}; + + for (CChainState* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + bool is_ibd = chainman.IsBackgroundIBD(chainstate); + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + if (coinscache.HaveCoin(op)) { + (is_ibd ? coins_in_ibd : coins_in_active)++; + } else if (is_ibd) { + coins_missing_ibd++; + } + } + } + + BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); + BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + } + + // Snapshot should refuse to load after one has already loaded. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Snapshot blockhash should be unchanged. + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + loaded_snapshot_blockhash); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 9e37f14921..ecf9453094 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <net.h> #include <signet.h> +#include <uint256.h> #include <validation.h> #include <test/util/setup_common.h> @@ -119,4 +120,27 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) BOOST_CHECK(!CheckSignetBlockSolution(block, signet_params->GetConsensus())); } +//! Test retrieval of valid assumeutxo values. +BOOST_AUTO_TEST_CASE(test_assumeutxo) +{ + const auto params = CreateChainParams(*m_node.args, CBaseChainParams::REGTEST); + + // These heights don't have assumeutxo configurations associated, per the contents + // of chainparams.cpp. + std::vector<int> bad_heights{0, 100, 111, 115, 209, 211}; + + for (auto empty : bad_heights) { + const auto out = ExpectedAssumeutxo(empty, *params); + BOOST_CHECK(!out); + } + + const auto out110 = *ExpectedAssumeutxo(110, *params); + BOOST_CHECK_EQUAL(out110.hash_serialized, uint256S("76fd7334ac7c1baf57ddc0c626f073a655a35d98a4258cd1382c8cc2b8392e10")); + BOOST_CHECK_EQUAL(out110.nChainTx, (unsigned int)110); + + const auto out210 = *ExpectedAssumeutxo(210, *params); + BOOST_CHECK_EQUAL(out210.hash_serialized, uint256S("9c5ed99ef98544b34f8920b6d1802f72ac28ae6e2bd2bd4c316ff10c230df3f2")); + BOOST_CHECK_EQUAL(out210.nChainTx, (unsigned int)210); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 90ee9422ba..0f2a4895f1 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -12,13 +12,16 @@ #include <net.h> #include <netaddress.h> #include <netbase.h> +#include <util/readwritefile.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/time.h> -#include <vector> #include <deque> +#include <functional> #include <set> #include <stdlib.h> +#include <vector> #include <boost/signals2/signal.hpp> #include <boost/algorithm/string/split.hpp> @@ -360,52 +363,6 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) return mapping; } -/** Read full contents of a file and return them in a std::string. - * Returns a pair <status, string>. - * If an error occurred, status will be false, otherwise status will be true and the data will be returned in string. - * - * @param maxsize Puts a maximum size limit on the file that is read. If the file is larger than this, truncated data - * (with len > maxsize) will be returned. - */ -static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits<size_t>::max()) -{ - FILE *f = fsbridge::fopen(filename, "rb"); - if (f == nullptr) - return std::make_pair(false,""); - std::string retval; - char buffer[128]; - size_t n; - while ((n=fread(buffer, 1, sizeof(buffer), f)) > 0) { - // Check for reading errors so we don't return any data if we couldn't - // read the entire file (or up to maxsize) - if (ferror(f)) { - fclose(f); - return std::make_pair(false,""); - } - retval.append(buffer, buffer+n); - if (retval.size() > maxsize) - break; - } - fclose(f); - return std::make_pair(true,retval); -} - -/** Write contents of std::string to a file. - * @return true on success. - */ -static bool WriteBinaryFile(const fs::path &filename, const std::string &data) -{ - FILE *f = fsbridge::fopen(filename, "wb"); - if (f == nullptr) - return false; - if (fwrite(data.data(), 1, data.size(), f) != data.size()) { - fclose(f); - return false; - } - fclose(f); - return true; -} - /****** Bitcoin specific TorController implementation ********/ /** Controller that connects to Tor control socket, authenticate, then create diff --git a/src/txdb.cpp b/src/txdb.cpp index 72460e7c69..4b4766e1ba 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -47,11 +47,15 @@ CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, b void CCoinsViewDB::ResizeCache(size_t new_cache_size) { - // Have to do a reset first to get the original `m_db` state to release its - // filesystem lock. - m_db.reset(); - m_db = MakeUnique<CDBWrapper>( - m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + // We can't do this operation with an in-memory DB since we'll lose all the coins upon + // reset. + if (!m_is_memory) { + // Have to do a reset first to get the original `m_db` state to release its + // filesystem lock. + m_db.reset(); + m_db = MakeUnique<CDBWrapper>( + m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + } } bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { diff --git a/src/txmempool.cpp b/src/txmempool.cpp index d18182c07d..48424a75d0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -9,21 +9,21 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <optional.h> -#include <validation.h> -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <policy/settings.h> #include <reverse_iterator.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <util/time.h> +#include <validation.h> #include <validationinterface.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp) : tx(_tx), nFee(_nFee), nTxWeight(GetTransactionWeight(*tx)), nUsageSize(RecursiveDynamicUsage(tx)), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp), m_epoch(0) + spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -132,7 +132,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes // include them, and update their CTxMemPoolEntry::m_parents to include this tx. // we cache the in-mempool children to avoid duplicate updates { - const auto epoch = GetFreshEpoch(); + WITH_FRESH_EPOCH(m_epoch); for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { const uint256 &childHash = iter->second->GetHash(); txiter childIter = mapTx.find(childHash); @@ -396,7 +396,10 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} + m_total_fee += entry.GetFee(); + if (minerPolicyEstimator) { + minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + } vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -432,6 +435,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) vTxHashes.clear(); totalTxSize -= it->GetTxSize(); + m_total_fee -= it->GetFee(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); mapTx.erase(it); @@ -499,7 +503,7 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso RemoveStaged(setAllRemoves, false, reason); } -void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags) +void CTxMemPool::removeForReorg(CChainState& active_chainstate, int flags) { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions AssertLockHeld(cs); @@ -507,8 +511,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { const CTransaction& tx = it->GetTx(); LockPoints lp = it->GetLockPoints(); - bool validLP = TestLockPointValidity(&lp); - if (!CheckFinalTx(tx, flags) || !CheckSequenceLocks(*this, tx, flags, &lp, validLP)) { + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); + bool validLP = TestLockPointValidity(active_chainstate.m_chain, &lp); + if (!CheckFinalTx(active_chainstate.m_chain.Tip(), tx, flags) || !CheckSequenceLocks(active_chainstate, *this, tx, flags, &lp, validLP)) { // Note if CheckSequenceLocks fails the LockPoints may still be invalid // So it's critical that we remove the tx and not depend on the LockPoints. txToRemove.insert(it); @@ -517,8 +522,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const Coin &coin = pcoins->AccessCoin(txin.prevout); + const Coin &coin = active_chainstate.CoinsTip().AccessCoin(txin.prevout); if (m_check_ratio != 0) assert(!coin.IsSpent()); + unsigned int nMemPoolHeight = active_chainstate.m_chain.Tip()->nHeight + 1; if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; @@ -590,6 +596,7 @@ void CTxMemPool::_clear() mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; + m_total_fee = 0; cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; @@ -618,19 +625,22 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(m_check_ratio) >= 1) return; + AssertLockHeld(::cs_main); LOCK(cs); LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; + CAmount check_total_fee{0}; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); - const int64_t spendheight = GetSpendHeight(mempoolDuplicate); + const int64_t spendheight = g_chainman.m_blockman.GetSpendHeight(mempoolDuplicate); std::list<const CTxMemPoolEntry*> waitingOnDependants; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->GetTxSize(); + check_total_fee += it->GetFee(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); @@ -725,6 +735,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const } assert(totalTxSize == checkTotal); + assert(m_total_fee == check_total_fee); assert(innerUsage == cachedInnerUsage); } @@ -1108,24 +1119,3 @@ void CTxMemPool::SetIsLoaded(bool loaded) LOCK(cs); m_is_loaded = loaded; } - - -CTxMemPool::EpochGuard CTxMemPool::GetFreshEpoch() const -{ - return EpochGuard(*this); -} -CTxMemPool::EpochGuard::EpochGuard(const CTxMemPool& in) : pool(in) -{ - assert(!pool.m_has_epoch_guard); - ++pool.m_epoch; - pool.m_has_epoch_guard = true; -} - -CTxMemPool::EpochGuard::~EpochGuard() -{ - // prevents stale results being used - ++pool.m_epoch; - pool.m_has_epoch_guard = false; -} - -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 15797cbc00..143048b205 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -15,13 +15,14 @@ #include <amount.h> #include <coins.h> -#include <crypto/siphash.h> #include <indirectmap.h> #include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> -#include <sync.h> #include <random.h> +#include <sync.h> +#include <util/epochguard.h> +#include <util/hasher.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/hashed_index.hpp> @@ -29,6 +30,7 @@ #include <boost/multi_index/sequenced_index.hpp> class CBlockIndex; +class CChainState; extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ @@ -63,6 +65,7 @@ struct CompareIteratorByHash { return a->GetTx().GetHash() < b->GetTx().GetHash(); } }; + /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well @@ -155,7 +158,7 @@ public: Children& GetMemPoolChildren() const { return m_children; } mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes - mutable uint64_t m_epoch; //!< epoch when last touched, useful for graph algorithms + mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -398,20 +401,6 @@ enum class MemPoolRemovalReason { REPLACED, //!< Removed for replacement }; -class SaltedTxidHasher -{ -private: - /** Salt */ - const uint64_t k0, k1; - -public: - SaltedTxidHasher(); - - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); - } -}; - /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -492,14 +481,14 @@ private: std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; - uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. - uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) + uint64_t totalTxSize GUARDED_BY(cs); //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. + CAmount m_total_fee GUARDED_BY(cs); //!< sum of all mempool tx's fees (NOT modified fee) + uint64_t cachedInnerUsage GUARDED_BY(cs); //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; mutable double rollingMinimumFeeRate; //!< minimum fee to get into the pool, decreases exponentially - mutable uint64_t m_epoch{0}; - mutable bool m_has_epoch_guard{false}; + mutable Epoch m_epoch GUARDED_BY(cs); // In-memory counter for external mempool tracking purposes. // This number is incremented once every time a transaction @@ -616,7 +605,7 @@ public: * all inputs are in the mapNextTx array). If sanity-checking is turned off, * check does nothing. */ - void check(const CCoinsViewCache *pcoins) const; + void check(const CCoinsViewCache *pcoins) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of @@ -629,7 +618,7 @@ public: void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); - void removeForReorg(const CCoinsViewCache* pcoins, unsigned int nMemPoolHeight, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + void removeForReorg(CChainState& active_chainstate, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeConflicts(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(cs); void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -678,7 +667,7 @@ public: * for). Note: vHashesToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ - void UpdateTransactionsFromBlock(const std::vector<uint256>& vHashesToUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + void UpdateTransactionsFromBlock(const std::vector<uint256>& vHashesToUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main) LOCKS_EXCLUDED(m_epoch); /** Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) @@ -738,6 +727,12 @@ public: return totalTxSize; } + CAmount GetTotalFee() const EXCLUSIVE_LOCKS_REQUIRED(cs) + { + AssertLockHeld(cs); + return m_total_fee; + } + bool exists(const GenTxid& gtxid) const { LOCK(cs); @@ -833,52 +828,22 @@ private: */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); public: - /** EpochGuard: RAII-style guard for using epoch-based graph traversal algorithms. - * When walking ancestors or descendants, we generally want to avoid - * visiting the same transactions twice. Some traversal algorithms use - * std::set (or setEntries) to deduplicate the transaction we visit. - * However, use of std::set is algorithmically undesirable because it both - * adds an asymptotic factor of O(log n) to traverals cost and triggers O(n) - * more dynamic memory allocations. - * In many algorithms we can replace std::set with an internal mempool - * counter to track the time (or, "epoch") that we began a traversal, and - * check + update a per-transaction epoch for each transaction we look at to - * determine if that transaction has not yet been visited during the current - * traversal's epoch. - * Algorithms using std::set can be replaced on a one by one basis. - * Both techniques are not fundamentally incompatible across the codebase. - * Generally speaking, however, the remaining use of std::set for mempool - * traversal should be viewed as a TODO for replacement with an epoch based - * traversal, rather than a preference for std::set over epochs in that - * algorithm. - */ - class EpochGuard { - const CTxMemPool& pool; - public: - explicit EpochGuard(const CTxMemPool& in); - ~EpochGuard(); - }; - // N.B. GetFreshEpoch modifies mutable state via the EpochGuard construction - // (and later destruction) - EpochGuard GetFreshEpoch() const EXCLUSIVE_LOCKS_REQUIRED(cs); - /** visited marks a CTxMemPoolEntry as having been traversed - * during the lifetime of the most recently created EpochGuard + * during the lifetime of the most recently created Epoch::Guard * and returns false if we are the first visitor, true otherwise. * - * An EpochGuard must be held when visited is called or an assert will be + * An Epoch::Guard must be held when visited is called or an assert will be * triggered. * */ - bool visited(txiter it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { - assert(m_has_epoch_guard); - bool ret = it->m_epoch >= m_epoch; - it->m_epoch = std::max(it->m_epoch, m_epoch); - return ret; + bool visited(const txiter it) const EXCLUSIVE_LOCKS_REQUIRED(cs, m_epoch) + { + return m_epoch.visited(it->m_epoch_marker); } - bool visited(Optional<txiter> it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { - assert(m_has_epoch_guard); + bool visited(Optional<txiter> it) const EXCLUSIVE_LOCKS_REQUIRED(cs, m_epoch) + { + assert(m_epoch.guarded()); // verify guard even when it==nullopt return !it || visited(*it); } }; diff --git a/src/util/epochguard.h b/src/util/epochguard.h new file mode 100644 index 0000000000..1570ec4eb4 --- /dev/null +++ b/src/util/epochguard.h @@ -0,0 +1,91 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_EPOCHGUARD_H +#define BITCOIN_UTIL_EPOCHGUARD_H + +#include <threadsafety.h> + +#include <cassert> + +/** Epoch: RAII-style guard for using epoch-based graph traversal algorithms. + * When walking ancestors or descendants, we generally want to avoid + * visiting the same transactions twice. Some traversal algorithms use + * std::set (or setEntries) to deduplicate the transaction we visit. + * However, use of std::set is algorithmically undesirable because it both + * adds an asymptotic factor of O(log n) to traversals cost and triggers O(n) + * more dynamic memory allocations. + * In many algorithms we can replace std::set with an internal mempool + * counter to track the time (or, "epoch") that we began a traversal, and + * check + update a per-transaction epoch for each transaction we look at to + * determine if that transaction has not yet been visited during the current + * traversal's epoch. + * Algorithms using std::set can be replaced on a one by one basis. + * Both techniques are not fundamentally incompatible across the codebase. + * Generally speaking, however, the remaining use of std::set for mempool + * traversal should be viewed as a TODO for replacement with an epoch based + * traversal, rather than a preference for std::set over epochs in that + * algorithm. + */ + +class LOCKABLE Epoch +{ +private: + uint64_t m_raw_epoch = 0; + bool m_guarded = false; + +public: + Epoch() = default; + Epoch(const Epoch&) = delete; + Epoch& operator=(const Epoch&) = delete; + + bool guarded() const { return m_guarded; } + + class Marker + { + private: + uint64_t m_marker = 0; + + // only allow modification via Epoch member functions + friend class Epoch; + Marker& operator=(const Marker&) = delete; + }; + + class SCOPED_LOCKABLE Guard + { + private: + Epoch& m_epoch; + + public: + explicit Guard(Epoch& epoch) EXCLUSIVE_LOCK_FUNCTION(epoch) : m_epoch(epoch) + { + assert(!m_epoch.m_guarded); + ++m_epoch.m_raw_epoch; + m_epoch.m_guarded = true; + } + ~Guard() UNLOCK_FUNCTION() + { + assert(m_epoch.m_guarded); + ++m_epoch.m_raw_epoch; // ensure clear separation between epochs + m_epoch.m_guarded = false; + } + }; + + bool visited(Marker& marker) const EXCLUSIVE_LOCKS_REQUIRED(*this) + { + assert(m_guarded); + if (marker.m_marker < m_raw_epoch) { + // marker is from a previous epoch, so this is its first visit + marker.m_marker = m_raw_epoch; + return false; + } else { + return true; + } + } +}; + +#define WITH_FRESH_EPOCH(epoch) const Epoch::Guard PASTE2(epoch_guard_, __COUNTER__)(epoch) + +#endif // BITCOIN_UTIL_EPOCHGUARD_H diff --git a/src/util/error.cpp b/src/util/error.cpp index 76fac4d391..48c81693f3 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -31,6 +31,10 @@ bilingual_str TransactionErrorString(const TransactionError err) return Untranslated("Specified sighash value does not match value stored in PSBT"); case TransactionError::MAX_FEE_EXCEEDED: return Untranslated("Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)"); + case TransactionError::EXTERNAL_SIGNER_NOT_FOUND: + return Untranslated("External signer not found"); + case TransactionError::EXTERNAL_SIGNER_FAILED: + return Untranslated("External signer failed to sign"); // no default case, so the compiler can warn about missing cases } assert(false); diff --git a/src/util/error.h b/src/util/error.h index 6633498d2b..4cc35eb1fd 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -30,6 +30,8 @@ enum class TransactionError { PSBT_MISMATCH, SIGHASH_MISMATCH, MAX_FEE_EXCEEDED, + EXTERNAL_SIGNER_NOT_FOUND, + EXTERNAL_SIGNER_FAILED, }; bilingual_str TransactionErrorString(const TransactionError error); diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp new file mode 100644 index 0000000000..9839d2f624 --- /dev/null +++ b/src/util/getuniquepath.cpp @@ -0,0 +1,10 @@ +#include <random.h> +#include <fs.h> +#include <util/strencodings.h> + +fs::path GetUniquePath(const fs::path& base) +{ + FastRandomContext rnd; + fs::path tmpFile = base / HexStr(rnd.randbytes(8)); + return tmpFile; +}
\ No newline at end of file diff --git a/src/util/getuniquepath.h b/src/util/getuniquepath.h new file mode 100644 index 0000000000..e0c6147876 --- /dev/null +++ b/src/util/getuniquepath.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GETUNIQUEPATH_H +#define BITCOIN_UTIL_GETUNIQUEPATH_H + +#include <fs.h> + +/** + * Helper function for getting a unique path + * + * @param[in] base Base path + * @returns base joined with a random 8-character long string. + * @post Returned path is unique with high probability. + */ +fs::path GetUniquePath(const fs::path& base); + +#endif // BITCOIN_UTIL_GETUNIQUEPATH_H
\ No newline at end of file diff --git a/src/util/hasher.cpp b/src/util/hasher.cpp new file mode 100644 index 0000000000..5900daf050 --- /dev/null +++ b/src/util/hasher.cpp @@ -0,0 +1,19 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <random.h> +#include <util/hasher.h> + +#include <limits> + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +SaltedSipHasher::SaltedSipHasher() : m_k0(GetRand(std::numeric_limits<uint64_t>::max())), m_k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +size_t SaltedSipHasher::operator()(const Span<const unsigned char>& script) const +{ + return CSipHasher(m_k0, m_k1).Write(script.data(), script.size()).Finalize(); +} diff --git a/src/util/hasher.h b/src/util/hasher.h new file mode 100644 index 0000000000..fa2fea30d8 --- /dev/null +++ b/src/util/hasher.h @@ -0,0 +1,99 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_HASHER_H +#define BITCOIN_UTIL_HASHER_H + +#include <crypto/siphash.h> +#include <primitives/transaction.h> +#include <uint256.h> + +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + +class SaltedOutpointHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedOutpointHasher(); + + /** + * This *must* return size_t. With Boost 1.46 on 32-bit systems the + * unordered_map will behave unpredictably if the custom hasher returns a + * uint64_t, resulting in failures when syncing the chain (#4634). + * + * Having the hash noexcept allows libstdc++'s unordered_map to recalculate + * the hash during rehash, so it does not have to cache the value. This + * reduces node's memory by sizeof(size_t). The required recalculation has + * a slight performance penalty (around 1.6%), but this is compensated by + * memory savings of about 9% which allow for a larger dbcache setting. + * + * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html + */ + size_t operator()(const COutPoint& id) const noexcept { + return SipHashUint256Extra(k0, k1, id.hash, id.n); + } +}; + +struct FilterHeaderHasher +{ + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + +struct BlockHasher +{ + // this used to call `GetCheapHash()` in uint256, which was later moved; the + // cheap hash function simply calls ReadLE64() however, so the end result is + // identical + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + +class SaltedSipHasher +{ +private: + /** Salt */ + const uint64_t m_k0, m_k1; + +public: + SaltedSipHasher(); + + size_t operator()(const Span<const unsigned char>& script) const; +}; + +#endif // BITCOIN_UTIL_HASHER_H diff --git a/src/util/macros.h b/src/util/macros.h index 36ea87c0fe..0887c80fd7 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -8,4 +8,11 @@ #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) +/** + * Converts the parameter X to a string after macro replacement on X has been performed. + * Don't merge these into one macro! + */ +#define STRINGIZE(X) DO_STRINGIZE(X) +#define DO_STRINGIZE(X) #X + #endif // BITCOIN_UTIL_MACROS_H diff --git a/src/util/message.cpp b/src/util/message.cpp index e1d5cff48c..73948e4ff1 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -31,7 +31,7 @@ MessageVerificationResult MessageVerify( return MessageVerificationResult::ERR_INVALID_ADDRESS; } - if (boost::get<PKHash>(&destination) == nullptr) { + if (std::get_if<PKHash>(&destination) == nullptr) { return MessageVerificationResult::ERR_ADDRESS_NO_KEY; } diff --git a/src/util/readwritefile.cpp b/src/util/readwritefile.cpp new file mode 100644 index 0000000000..a45c41d367 --- /dev/null +++ b/src/util/readwritefile.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> + +#include <limits> +#include <stdio.h> +#include <string> +#include <utility> + +std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits<size_t>::max()) +{ + FILE *f = fsbridge::fopen(filename, "rb"); + if (f == nullptr) + return std::make_pair(false,""); + std::string retval; + char buffer[128]; + do { + const size_t n = fread(buffer, 1, sizeof(buffer), f); + // Check for reading errors so we don't return any data if we couldn't + // read the entire file (or up to maxsize) + if (ferror(f)) { + fclose(f); + return std::make_pair(false,""); + } + retval.append(buffer, buffer+n); + } while (!feof(f) && retval.size() <= maxsize); + fclose(f); + return std::make_pair(true,retval); +} + +bool WriteBinaryFile(const fs::path &filename, const std::string &data) +{ + FILE *f = fsbridge::fopen(filename, "wb"); + if (f == nullptr) + return false; + if (fwrite(data.data(), 1, data.size(), f) != data.size()) { + fclose(f); + return false; + } + if (fclose(f) != 0) { + return false; + } + return true; +} diff --git a/src/util/readwritefile.h b/src/util/readwritefile.h new file mode 100644 index 0000000000..1dab874b38 --- /dev/null +++ b/src/util/readwritefile.h @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_READWRITEFILE_H +#define BITCOIN_UTIL_READWRITEFILE_H + +#include <fs.h> + +#include <limits> +#include <string> +#include <utility> + +/** Read full contents of a file and return them in a std::string. + * Returns a pair <status, string>. + * If an error occurred, status will be false, otherwise status will be true and the data will be returned in string. + * + * @param maxsize Puts a maximum size limit on the file that is read. If the file is larger than this, truncated data + * (with len > maxsize) will be returned. + */ +std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxsize=std::numeric_limits<size_t>::max()); + +/** Write contents of std::string to a file. + * @return true on success. + */ +bool WriteBinaryFile(const fs::path &filename, const std::string &data); + +#endif /* BITCOIN_UTIL_READWRITEFILE_H */ diff --git a/src/util/sock.cpp b/src/util/sock.cpp new file mode 100644 index 0000000000..e13c52a16a --- /dev/null +++ b/src/util/sock.cpp @@ -0,0 +1,327 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <compat.h> +#include <logging.h> +#include <threadinterrupt.h> +#include <tinyformat.h> +#include <util/sock.h> +#include <util/system.h> +#include <util/time.h> + +#include <codecvt> +#include <cwchar> +#include <locale> +#include <stdexcept> +#include <string> + +#ifdef USE_POLL +#include <poll.h> +#endif + +static inline bool IOErrorIsPermanent(int err) +{ + return err != WSAEAGAIN && err != WSAEINTR && err != WSAEWOULDBLOCK && err != WSAEINPROGRESS; +} + +Sock::Sock() : m_socket(INVALID_SOCKET) {} + +Sock::Sock(SOCKET s) : m_socket(s) {} + +Sock::Sock(Sock&& other) +{ + m_socket = other.m_socket; + other.m_socket = INVALID_SOCKET; +} + +Sock::~Sock() { Reset(); } + +Sock& Sock::operator=(Sock&& other) +{ + Reset(); + m_socket = other.m_socket; + other.m_socket = INVALID_SOCKET; + return *this; +} + +SOCKET Sock::Get() const { return m_socket; } + +SOCKET Sock::Release() +{ + const SOCKET s = m_socket; + m_socket = INVALID_SOCKET; + return s; +} + +void Sock::Reset() { CloseSocket(m_socket); } + +ssize_t Sock::Send(const void* data, size_t len, int flags) const +{ + return send(m_socket, static_cast<const char*>(data), len, flags); +} + +ssize_t Sock::Recv(void* buf, size_t len, int flags) const +{ + return recv(m_socket, static_cast<char*>(buf), len, flags); +} + +bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const +{ +#ifdef USE_POLL + pollfd fd; + fd.fd = m_socket; + fd.events = 0; + if (requested & RECV) { + fd.events |= POLLIN; + } + if (requested & SEND) { + fd.events |= POLLOUT; + } + + if (poll(&fd, 1, count_milliseconds(timeout)) == SOCKET_ERROR) { + return false; + } + + if (occurred != nullptr) { + *occurred = 0; + if (fd.revents & POLLIN) { + *occurred |= RECV; + } + if (fd.revents & POLLOUT) { + *occurred |= SEND; + } + } + + return true; +#else + if (!IsSelectableSocket(m_socket)) { + return false; + } + + fd_set fdset_recv; + fd_set fdset_send; + FD_ZERO(&fdset_recv); + FD_ZERO(&fdset_send); + + if (requested & RECV) { + FD_SET(m_socket, &fdset_recv); + } + + if (requested & SEND) { + FD_SET(m_socket, &fdset_send); + } + + timeval timeout_struct = MillisToTimeval(timeout); + + if (select(m_socket + 1, &fdset_recv, &fdset_send, nullptr, &timeout_struct) == SOCKET_ERROR) { + return false; + } + + if (occurred != nullptr) { + *occurred = 0; + if (FD_ISSET(m_socket, &fdset_recv)) { + *occurred |= RECV; + } + if (FD_ISSET(m_socket, &fdset_send)) { + *occurred |= SEND; + } + } + + return true; +#endif /* USE_POLL */ +} + +void Sock::SendComplete(const std::string& data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + const auto deadline = GetTime<std::chrono::milliseconds>() + timeout; + size_t sent{0}; + + for (;;) { + const ssize_t ret{Send(data.data() + sent, data.size() - sent, MSG_NOSIGNAL)}; + + if (ret > 0) { + sent += static_cast<size_t>(ret); + if (sent == data.size()) { + break; + } + } else { + const int err{WSAGetLastError()}; + if (IOErrorIsPermanent(err)) { + throw std::runtime_error(strprintf("send(): %s", NetworkErrorString(err))); + } + } + + const auto now = GetTime<std::chrono::milliseconds>(); + + if (now >= deadline) { + throw std::runtime_error(strprintf( + "Send timeout (sent only %u of %u bytes before that)", sent, data.size())); + } + + if (interrupt) { + throw std::runtime_error(strprintf( + "Send interrupted (sent only %u of %u bytes before that)", sent, data.size())); + } + + // Wait for a short while (or the socket to become ready for sending) before retrying + // if nothing was sent. + const auto wait_time = std::min(deadline - now, std::chrono::milliseconds{MAX_WAIT_FOR_IO}); + Wait(wait_time, SEND); + } +} + +std::string Sock::RecvUntilTerminator(uint8_t terminator, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + const auto deadline = GetTime<std::chrono::milliseconds>() + timeout; + std::string data; + bool terminator_found{false}; + + // We must not consume any bytes past the terminator from the socket. + // One option is to read one byte at a time and check if we have read a terminator. + // However that is very slow. Instead, we peek at what is in the socket and only read + // as many bytes as possible without crossing the terminator. + // Reading 64 MiB of random data with 262526 terminator chars takes 37 seconds to read + // one byte at a time VS 0.71 seconds with the "peek" solution below. Reading one byte + // at a time is about 50 times slower. + + for (;;) { + char buf[512]; + + const ssize_t peek_ret{Recv(buf, sizeof(buf), MSG_PEEK)}; + + switch (peek_ret) { + case -1: { + const int err{WSAGetLastError()}; + if (IOErrorIsPermanent(err)) { + throw std::runtime_error(strprintf("recv(): %s", NetworkErrorString(err))); + } + break; + } + case 0: + throw std::runtime_error("Connection unexpectedly closed by peer"); + default: + auto end = buf + peek_ret; + auto terminator_pos = std::find(buf, end, terminator); + terminator_found = terminator_pos != end; + + const size_t try_len{terminator_found ? terminator_pos - buf + 1 : + static_cast<size_t>(peek_ret)}; + + const ssize_t read_ret{Recv(buf, try_len, 0)}; + + if (read_ret < 0 || static_cast<size_t>(read_ret) != try_len) { + throw std::runtime_error( + strprintf("recv() returned %u bytes on attempt to read %u bytes but previous " + "peek claimed %u bytes are available", + read_ret, try_len, peek_ret)); + } + + // Don't include the terminator in the output. + const size_t append_len{terminator_found ? try_len - 1 : try_len}; + + data.append(buf, buf + append_len); + + if (terminator_found) { + return data; + } + } + + const auto now = GetTime<std::chrono::milliseconds>(); + + if (now >= deadline) { + throw std::runtime_error(strprintf( + "Receive timeout (received %u bytes without terminator before that)", data.size())); + } + + if (interrupt) { + throw std::runtime_error(strprintf( + "Receive interrupted (received %u bytes without terminator before that)", + data.size())); + } + + // Wait for a short while (or the socket to become ready for reading) before retrying. + const auto wait_time = std::min(deadline - now, std::chrono::milliseconds{MAX_WAIT_FOR_IO}); + Wait(wait_time, RECV); + } +} + +bool Sock::IsConnected(std::string& errmsg) const +{ + if (m_socket == INVALID_SOCKET) { + errmsg = "not connected"; + return false; + } + + char c; + switch (Recv(&c, sizeof(c), MSG_PEEK)) { + case -1: { + const int err = WSAGetLastError(); + if (IOErrorIsPermanent(err)) { + errmsg = NetworkErrorString(err); + return false; + } + return true; + } + case 0: + errmsg = "closed"; + return false; + default: + return true; + } +} + +#ifdef WIN32 +std::string NetworkErrorString(int err) +{ + wchar_t buf[256]; + buf[0] = 0; + if(FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, + nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buf, ARRAYSIZE(buf), nullptr)) + { + return strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err); + } + else + { + return strprintf("Unknown error (%d)", err); + } +} +#else +std::string NetworkErrorString(int err) +{ + char buf[256]; + buf[0] = 0; + /* Too bad there are two incompatible implementations of the + * thread-safe strerror. */ + const char *s; +#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ + s = strerror_r(err, buf, sizeof(buf)); +#else /* POSIX variant always returns message in buffer */ + s = buf; + if (strerror_r(err, buf, sizeof(buf))) + buf[0] = 0; +#endif + return strprintf("%s (%d)", s, err); +} +#endif + +bool CloseSocket(SOCKET& hSocket) +{ + if (hSocket == INVALID_SOCKET) + return false; +#ifdef WIN32 + int ret = closesocket(hSocket); +#else + int ret = close(hSocket); +#endif + if (ret) { + LogPrintf("Socket close failed: %d. Error: %s\n", hSocket, NetworkErrorString(WSAGetLastError())); + } + hSocket = INVALID_SOCKET; + return ret != SOCKET_ERROR; +} diff --git a/src/util/sock.h b/src/util/sock.h new file mode 100644 index 0000000000..ecebb84205 --- /dev/null +++ b/src/util/sock.h @@ -0,0 +1,166 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SOCK_H +#define BITCOIN_UTIL_SOCK_H + +#include <compat.h> +#include <threadinterrupt.h> +#include <util/time.h> + +#include <chrono> +#include <string> + +/** + * Maximum time to wait for I/O readiness. + * It will take up until this time to break off in case of an interruption. + */ +static constexpr auto MAX_WAIT_FOR_IO = 1s; + +/** + * RAII helper class that manages a socket. Mimics `std::unique_ptr`, but instead of a pointer it + * contains a socket and closes it automatically when it goes out of scope. + */ +class Sock +{ +public: + /** + * Default constructor, creates an empty object that does nothing when destroyed. + */ + Sock(); + + /** + * Take ownership of an existent socket. + */ + explicit Sock(SOCKET s); + + /** + * Copy constructor, disabled because closing the same socket twice is undesirable. + */ + Sock(const Sock&) = delete; + + /** + * Move constructor, grab the socket from another object and close ours (if set). + */ + Sock(Sock&& other); + + /** + * Destructor, close the socket or do nothing if empty. + */ + virtual ~Sock(); + + /** + * Copy assignment operator, disabled because closing the same socket twice is undesirable. + */ + Sock& operator=(const Sock&) = delete; + + /** + * Move assignment operator, grab the socket from another object and close ours (if set). + */ + virtual Sock& operator=(Sock&& other); + + /** + * Get the value of the contained socket. + * @return socket or INVALID_SOCKET if empty + */ + virtual SOCKET Get() const; + + /** + * Get the value of the contained socket and drop ownership. It will not be closed by the + * destructor after this call. + * @return socket or INVALID_SOCKET if empty + */ + virtual SOCKET Release(); + + /** + * Close if non-empty. + */ + virtual void Reset(); + + /** + * send(2) wrapper. Equivalent to `send(this->Get(), data, len, flags);`. Code that uses this + * wrapper can be unit-tested if this method is overridden by a mock Sock implementation. + */ + virtual ssize_t Send(const void* data, size_t len, int flags) const; + + /** + * recv(2) wrapper. Equivalent to `recv(this->Get(), buf, len, flags);`. Code that uses this + * wrapper can be unit-tested if this method is overridden by a mock Sock implementation. + */ + virtual ssize_t Recv(void* buf, size_t len, int flags) const; + + using Event = uint8_t; + + /** + * If passed to `Wait()`, then it will wait for readiness to read from the socket. + */ + static constexpr Event RECV = 0b01; + + /** + * If passed to `Wait()`, then it will wait for readiness to send to the socket. + */ + static constexpr Event SEND = 0b10; + + /** + * Wait for readiness for input (recv) or output (send). + * @param[in] timeout Wait this much for at least one of the requested events to occur. + * @param[in] requested Wait for those events, bitwise-or of `RECV` and `SEND`. + * @param[out] occurred If not nullptr and `true` is returned, then upon return this + * indicates which of the requested events occurred. A timeout is indicated by return + * value of `true` and `occurred` being set to 0. + * @return true on success and false otherwise + */ + virtual bool Wait(std::chrono::milliseconds timeout, + Event requested, + Event* occurred = nullptr) const; + + /* Higher level, convenience, methods. These may throw. */ + + /** + * Send the given data, retrying on transient errors. + * @param[in] data Data to send. + * @param[in] timeout Timeout for the entire operation. + * @param[in] interrupt If this is signaled then the operation is canceled. + * @throws std::runtime_error if the operation cannot be completed. In this case only some of + * the data will be written to the socket. + */ + virtual void SendComplete(const std::string& data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Read from socket until a terminator character is encountered. Will never consume bytes past + * the terminator from the socket. + * @param[in] terminator Character up to which to read from the socket. + * @param[in] timeout Timeout for the entire operation. + * @param[in] interrupt If this is signaled then the operation is canceled. + * @return The data that has been read, without the terminating character. + * @throws std::runtime_error if the operation cannot be completed. In this case some bytes may + * have been consumed from the socket. + */ + virtual std::string RecvUntilTerminator(uint8_t terminator, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Check if still connected. + * @param[out] err The error string, if the socket has been disconnected. + * @return true if connected + */ + virtual bool IsConnected(std::string& errmsg) const; + +private: + /** + * Contained socket. `INVALID_SOCKET` designates the object is empty. + */ + SOCKET m_socket; +}; + +/** Return readable error string for a network error code */ +std::string NetworkErrorString(int err); + +/** Close socket and set hSocket to INVALID_SOCKET */ +bool CloseSocket(SOCKET& hSocket); + +#endif // BITCOIN_UTIL_SOCK_H diff --git a/src/util/strencodings.h b/src/util/strencodings.h index b4a61202ef..98379e9138 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -17,8 +17,6 @@ #include <string> #include <vector> -#define ARRAYLEN(array) (sizeof(array)/sizeof((array)[0])) - /** Used by SanitizeString() */ enum SafeChars { diff --git a/src/util/string.h b/src/util/string.h index 5ffdc80d88..b26facc502 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -25,6 +25,14 @@ return str.substr(front, end - front + 1); } +[[nodiscard]] inline std::string RemovePrefix(const std::string& str, const std::string& prefix) +{ + if (str.substr(0, prefix.size()) == prefix) { + return str.substr(prefix.size()); + } + return str; +} + /** * Join a list of items * diff --git a/src/util/system.cpp b/src/util/system.cpp index 917d37ccc7..71453eed81 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,14 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <sync.h> #include <util/system.h> -#ifdef HAVE_BOOST_PROCESS +#ifdef ENABLE_EXTERNAL_SIGNER #include <boost/process.hpp> -#endif // HAVE_BOOST_PROCESS +#endif // ENABLE_EXTERNAL_SIGNER #include <chainparamsbase.h> +#include <sync.h> +#include <util/check.h> +#include <util/getuniquepath.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -123,7 +125,7 @@ void ReleaseDirectoryLocks() bool DirIsWritable(const fs::path& directory) { - fs::path tmpFile = directory / fs::unique_path(); + fs::path tmpFile = GetUniquePath(directory); FILE* file = fsbridge::fopen(tmpFile, "a"); if (!file) return false; @@ -310,8 +312,22 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin key[0] = '-'; #endif - if (key[0] != '-') + if (key[0] != '-') { + if (!m_accept_any_command && m_command.empty()) { + // The first non-dash arg is a registered command + Optional<unsigned int> flags = GetArgFlags(key); + if (!flags || !(*flags & ArgsManager::COMMAND)) { + error = strprintf("Invalid command '%s'", argv[i]); + return false; + } + } + m_command.push_back(key); + while (++i < argc) { + // The remaining args are command args + m_command.push_back(argv[i]); + } break; + } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') @@ -359,6 +375,26 @@ Optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) const return nullopt; } +std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const +{ + Command ret; + LOCK(cs_args); + auto it = m_command.begin(); + if (it == m_command.end()) { + // No command was passed + return std::nullopt; + } + if (!m_accept_any_command) { + // The registered command + ret.command = *(it++); + } + while (it != m_command.end()) { + // The unregistered command and args (if any) + ret.args.push_back(*(it++)); + } + return ret; +} + std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; @@ -398,7 +434,7 @@ bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const } if (filepath) { std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); - *filepath = fs::absolute(temp ? settings + ".tmp" : settings, GetDataDir(/* net_specific= */ true)); + *filepath = fsbridge::AbsPathJoin(GetDataDir(/* net_specific= */ true), temp ? settings + ".tmp" : settings); } return true; } @@ -504,8 +540,22 @@ void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strV m_settings.forced_settings[SettingName(strArg)] = strValue; } +void ArgsManager::AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat) +{ + Assert(cmd.find('=') == std::string::npos); + Assert(cmd.at(0) != '-'); + + LOCK(cs_args); + m_accept_any_command = false; // latch to false + std::map<std::string, Arg>& arg_map = m_available_args[cat]; + auto ret = arg_map.emplace(cmd, Arg{"", help, ArgsManager::COMMAND}); + Assert(ret.second); // Fail on duplicate commands +} + void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) { + Assert((flags & ArgsManager::COMMAND) == 0); // use AddCommand + // Split arg name from its help param size_t eq_index = name.find('='); if (eq_index == std::string::npos) { @@ -1197,7 +1247,7 @@ void runCommand(const std::string& strCommand) } #endif -#ifdef HAVE_BOOST_PROCESS +#ifdef ENABLE_EXTERNAL_SIGNER UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) { namespace bp = boost::process; @@ -1232,7 +1282,7 @@ UniValue RunCommandParseJSON(const std::string& str_command, const std::string& return result_json; } -#endif // HAVE_BOOST_PROCESS +#endif // ENABLE_EXTERNAL_SIGNER void SetupEnvironment() { @@ -1311,7 +1361,7 @@ fs::path AbsPathForConfigVal(const fs::path& path, bool net_specific) if (path.is_absolute()) { return path; } - return fs::absolute(path, GetDataDir(net_specific)); + return fsbridge::AbsPathJoin(GetDataDir(net_specific), path); } void ScheduleBatchPriority() diff --git a/src/util/system.h b/src/util/system.h index 010fc5b49f..de47b93b6e 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -35,8 +35,6 @@ #include <utility> #include <vector> -#include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted - class UniValue; // Application startup time (used for uptime calculation) @@ -110,7 +108,7 @@ std::string ShellEscape(const std::string& arg); #if HAVE_SYSTEM void runCommand(const std::string& strCommand); #endif -#ifdef HAVE_BOOST_PROCESS +#ifdef ENABLE_EXTERNAL_SIGNER /** * Execute a command which returns JSON, and parse the result. * @@ -119,7 +117,7 @@ void runCommand(const std::string& strCommand); * @return parsed JSON */ UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in=""); -#endif // HAVE_BOOST_PROCESS +#endif // ENABLE_EXTERNAL_SIGNER /** * Most paths passed as configuration arguments are treated as relative to @@ -168,7 +166,7 @@ struct SectionInfo class ArgsManager { public: - enum Flags { + enum Flags : uint32_t { // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 ALLOW_BOOL = 0x01, ALLOW_INT = 0x02, @@ -183,6 +181,7 @@ public: NETWORK_ONLY = 0x200, // This argument's value is sensitive (such as a password). SENSITIVE = 0x400, + COMMAND = 0x800, }; protected: @@ -195,9 +194,11 @@ protected: mutable RecursiveMutex cs_args; util::Settings m_settings GUARDED_BY(cs_args); + std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); + bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); [[nodiscard]] bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); @@ -248,6 +249,20 @@ public: */ const std::list<SectionInfo> GetUnrecognizedSections() const; + struct Command { + /** The command (if one has been registered with AddCommand), or empty */ + std::string command; + /** + * If command is non-empty: Any args that followed it + * If command is empty: The unregistered command and any args that followed it + */ + std::vector<std::string> args; + }; + /** + * Get the command and command args (returns std::nullopt if no command provided) + */ + std::optional<const Command> GetCommand() const; + /** * Return a vector of strings of the given argument * @@ -334,6 +349,11 @@ public: void AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat); /** + * Add subcommand + */ + void AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat); + + /** * Add many hidden arguments */ void AddHiddenArgs(const std::vector<std::string>& args); @@ -450,11 +470,6 @@ template <typename Callable> void TraceThread(const char* name, Callable func) func(); LogPrintf("%s thread exit\n", name); } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread interrupt\n", name); - throw; - } catch (const std::exception& e) { PrintExceptionContinue(&e, name); throw; diff --git a/src/util/time.cpp b/src/util/time.cpp index e96972fe12..e6f0986a39 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -7,8 +7,11 @@ #include <config/bitcoin-config.h> #endif +#include <compat.h> #include <util/time.h> +#include <util/check.h> + #include <atomic> #include <boost/date_time/posix_time/posix_time.hpp> #include <ctime> @@ -18,7 +21,7 @@ void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } -static std::atomic<int64_t> nMockTime(0); //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For testing int64_t GetTime() { @@ -30,6 +33,49 @@ int64_t GetTime() return now; } +bool ChronoSanityCheck() +{ + // std::chrono::system_clock.time_since_epoch and time_t(0) are not guaranteed + // to use the Unix epoch timestamp, prior to C++20, but in practice they almost + // certainly will. Any differing behavior will be assumed to be an error, unless + // certain platforms prove to consistently deviate, at which point we'll cope + // with it by adding offsets. + + // Create a new clock from time_t(0) and make sure that it represents 0 + // seconds from the system_clock's time_since_epoch. Then convert that back + // to a time_t and verify that it's the same as before. + const time_t time_t_epoch{}; + auto clock = std::chrono::system_clock::from_time_t(time_t_epoch); + if (std::chrono::duration_cast<std::chrono::seconds>(clock.time_since_epoch()).count() != 0) { + return false; + } + + time_t time_val = std::chrono::system_clock::to_time_t(clock); + if (time_val != time_t_epoch) { + return false; + } + + // Check that the above zero time is actually equal to the known unix timestamp. + struct tm epoch; +#ifdef HAVE_GMTIME_R + if (gmtime_r(&time_val, &epoch) == nullptr) { +#else + if (gmtime_s(&epoch, &time_val) != 0) { +#endif + return false; + } + + if ((epoch.tm_sec != 0) || + (epoch.tm_min != 0) || + (epoch.tm_hour != 0) || + (epoch.tm_mday != 1) || + (epoch.tm_mon != 0) || + (epoch.tm_year != 70)) { + return false; + } + return true; +} + template <typename T> T GetTime() { @@ -44,35 +90,43 @@ template std::chrono::seconds GetTime(); template std::chrono::milliseconds GetTime(); template std::chrono::microseconds GetTime(); +template <typename T> +static T GetSystemTime() +{ + const auto now = std::chrono::duration_cast<T>(std::chrono::system_clock::now().time_since_epoch()); + assert(now.count() > 0); + return now; +} + void SetMockTime(int64_t nMockTimeIn) { + Assert(nMockTimeIn >= 0); nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } -int64_t GetMockTime() +void SetMockTime(std::chrono::seconds mock_time_in) +{ + nMockTime.store(mock_time_in.count(), std::memory_order_relaxed); +} + +std::chrono::seconds GetMockTime() { - return nMockTime.load(std::memory_order_relaxed); + return std::chrono::seconds(nMockTime.load(std::memory_order_relaxed)); } int64_t GetTimeMillis() { - int64_t now = (boost::posix_time::microsec_clock::universal_time() - - boost::posix_time::ptime(boost::gregorian::date(1970,1,1))).total_milliseconds(); - assert(now > 0); - return now; + return int64_t{GetSystemTime<std::chrono::milliseconds>().count()}; } int64_t GetTimeMicros() { - int64_t now = (boost::posix_time::microsec_clock::universal_time() - - boost::posix_time::ptime(boost::gregorian::date(1970,1,1))).total_microseconds(); - assert(now > 0); - return now; + return int64_t{GetSystemTime<std::chrono::microseconds>().count()}; } int64_t GetSystemTimeInSeconds() { - return GetTimeMicros()/1000000; + return int64_t{GetSystemTime<std::chrono::seconds>().count()}; } std::string FormatISO8601DateTime(int64_t nTime) { @@ -114,3 +168,16 @@ int64_t ParseISO8601DateTime(const std::string& str) return 0; return (ptime - epoch).total_seconds(); } + +struct timeval MillisToTimeval(int64_t nTimeout) +{ + struct timeval timeout; + timeout.tv_sec = nTimeout / 1000; + timeout.tv_usec = (nTimeout % 1000) * 1000; + return timeout; +} + +struct timeval MillisToTimeval(std::chrono::milliseconds ms) +{ + return MillisToTimeval(count_milliseconds(ms)); +} diff --git a/src/util/time.h b/src/util/time.h index c69f604dc6..56131ce0fe 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_UTIL_TIME_H #define BITCOIN_UTIL_TIME_H +#include <compat.h> + #include <chrono> #include <stdint.h> #include <string> @@ -25,6 +27,7 @@ void UninterruptibleSleep(const std::chrono::microseconds& n); * interface that doesn't support std::chrono (e.g. RPC, debug log, or the GUI) */ inline int64_t count_seconds(std::chrono::seconds t) { return t.count(); } +inline int64_t count_milliseconds(std::chrono::milliseconds t) { return t.count(); } inline int64_t count_microseconds(std::chrono::microseconds t) { return t.count(); } /** @@ -40,10 +43,19 @@ int64_t GetTimeMicros(); /** Returns the system time (not mockable) */ int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -/** For testing. Set e.g. with the setmocktime rpc, or -mocktime argument */ +/** + * DEPRECATED + * Use SetMockTime with chrono type + * + * @param[in] nMockTimeIn Time in seconds. + */ void SetMockTime(int64_t nMockTimeIn); + +/** For testing. Set e.g. with the setmocktime rpc, or -mocktime argument */ +void SetMockTime(std::chrono::seconds mock_time_in); + /** For testing */ -int64_t GetMockTime(); +std::chrono::seconds GetMockTime(); /** Return system time (or mocked time, if set) */ template <typename T> @@ -57,4 +69,17 @@ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); int64_t ParseISO8601DateTime(const std::string& str); +/** + * Convert milliseconds to a struct timeval for e.g. select. + */ +struct timeval MillisToTimeval(int64_t nTimeout); + +/** + * Convert milliseconds to a struct timeval for e.g. select. + */ +struct timeval MillisToTimeval(std::chrono::milliseconds ms); + +/** Sanity check epoch match normal Unix epoch */ +bool ChronoSanityCheck(); + #endif // BITCOIN_UTIL_TIME_H diff --git a/src/util/trace.h b/src/util/trace.h new file mode 100644 index 0000000000..9c92cb10e7 --- /dev/null +++ b/src/util/trace.h @@ -0,0 +1,45 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_TRACE_H +#define BITCOIN_UTIL_TRACE_H + +#ifdef ENABLE_TRACING + +#include <sys/sdt.h> + +#define TRACE(context, event) DTRACE_PROBE(context, event) +#define TRACE1(context, event, a) DTRACE_PROBE1(context, event, a) +#define TRACE2(context, event, a, b) DTRACE_PROBE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) DTRACE_PROBE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) DTRACE_PROBE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) DTRACE_PROBE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) DTRACE_PROBE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) + +#else + +#define TRACE(context, event) +#define TRACE1(context, event, a) +#define TRACE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) + +#endif + + +#endif /* BITCOIN_UTIL_TRACE_H */ diff --git a/src/validation.cpp b/src/validation.cpp index 4f7c95b2ad..0b2ca4b422 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -17,9 +17,11 @@ #include <cuckoocache.h> #include <flatfile.h> #include <hash.h> +#include <index/blockfilterindex.h> #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <node/coinstats.h> #include <node/ui_interface.h> #include <optional.h> #include <policy/policy.h> @@ -167,17 +169,19 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -CBlockIndex* LookupBlockIndex(const uint256& hash) +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); - return it == g_chainman.BlockIndex().end() ? nullptr : it->second; + assert(std::addressof(g_chainman.BlockIndex()) == std::addressof(m_block_index)); + BlockMap::const_iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : it->second; } -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { @@ -195,14 +199,19 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc std::unique_ptr<CBlockTreeDB> pblocktree; -bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); +bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, + const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, + bool cacheFullScriptStore, PrecomputedTransactionData& txdata, + std::vector<CScriptCheck>* pvChecks = nullptr) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); -bool CheckFinalTx(const CTransaction &tx, int flags) +bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) { AssertLockHeld(cs_main); + assert(std::addressof(*::ChainActive().Tip()) == std::addressof(*active_chain_tip)); // By convention a negative value for flags indicates that the // current network-enforced consensus rules should be used. In @@ -218,7 +227,7 @@ bool CheckFinalTx(const CTransaction &tx, int flags) // evaluated is what is used. Thus if we want to know if a // transaction can be part of the *next* block, we need to call // IsFinalTx() with one more than ::ChainActive().Height(). - const int nBlockHeight = ::ChainActive().Height() + 1; + const int nBlockHeight = active_chain_tip->nHeight + 1; // BIP113 requires that time-locked transactions have nLockTime set to // less than the median time of the previous block they're contained in. @@ -226,13 +235,13 @@ bool CheckFinalTx(const CTransaction &tx, int flags) // chain tip, so we use that to calculate the median time passed to // IsFinalTx() if LOCKTIME_MEDIAN_TIME_PAST is set. const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) - ? ::ChainActive().Tip()->GetMedianTimePast() + ? active_chain_tip->GetMedianTimePast() : GetAdjustedTime(); return IsFinalTx(tx, nBlockHeight, nBlockTime); } -bool TestLockPointValidity(const LockPoints* lp) +bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) { AssertLockHeld(cs_main); assert(lp); @@ -241,7 +250,8 @@ bool TestLockPointValidity(const LockPoints* lp) if (lp->maxInputBlock) { // Check whether ::ChainActive() is an extension of the block at which the LockPoints // calculation was valid. If not LockPoints are no longer valid - if (!::ChainActive().Contains(lp->maxInputBlock)) { + assert(std::addressof(::ChainActive()) == std::addressof(active_chain)); + if (!active_chain.Contains(lp->maxInputBlock)) { return false; } } @@ -250,22 +260,28 @@ bool TestLockPointValidity(const LockPoints* lp) return true; } -bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flags, LockPoints* lp, bool useExistingLockPoints) +bool CheckSequenceLocks(CChainState& active_chainstate, + const CTxMemPool& pool, + const CTransaction& tx, + int flags, + LockPoints* lp, + bool useExistingLockPoints) { AssertLockHeld(cs_main); AssertLockHeld(pool.cs); + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); - CBlockIndex* tip = ::ChainActive().Tip(); + CBlockIndex* tip = active_chainstate.m_chain.Tip(); assert(tip != nullptr); CBlockIndex index; index.pprev = tip; - // CheckSequenceLocks() uses ::ChainActive().Height()+1 to evaluate + // CheckSequenceLocks() uses active_chainstate.m_chain.Height()+1 to evaluate // height based locks because when SequenceLocks() is called within // ConnectBlock(), the height of the block *being* // evaluated is what is used. // Thus if we want to know if a transaction can be part of the - // *next* block, we need to use one more than ::ChainActive().Height() + // *next* block, we need to use one more than active_chainstate.m_chain.Height() index.nHeight = tip->nHeight + 1; std::pair<int, int64_t> lockPair; @@ -275,8 +291,8 @@ bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flag lockPair.second = lp->time; } else { - // CoinsTip() contains the UTXO set for ::ChainActive().Tip() - CCoinsViewMemPool viewMemPool(&::ChainstateActive().CoinsTip(), pool); + // CoinsTip() contains the UTXO set for active_chainstate.m_chain.Tip() + CCoinsViewMemPool viewMemPool(&active_chainstate.CoinsTip(), pool); std::vector<int> prevheights; prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { @@ -325,7 +341,7 @@ bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flag // Returns the script flags which should be checked for a given block static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& chainparams); -static void LimitMempoolSize(CTxMemPool& pool, size_t limit, std::chrono::seconds age) +static void LimitMempoolSize(CTxMemPool& pool, CCoinsViewCache& coins_cache, size_t limit, std::chrono::seconds age) EXCLUSIVE_LOCKS_REQUIRED(pool.cs, ::cs_main) { int expired = pool.Expire(GetTime<std::chrono::seconds>() - age); @@ -335,18 +351,20 @@ static void LimitMempoolSize(CTxMemPool& pool, size_t limit, std::chrono::second std::vector<COutPoint> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); + assert(std::addressof(::ChainstateActive().CoinsTip()) == std::addressof(coins_cache)); for (const COutPoint& removed : vNoSpendsRemaining) - ::ChainstateActive().CoinsTip().Uncache(removed); + coins_cache.Uncache(removed); } -static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); - if (::ChainstateActive().IsInitialBlockDownload()) + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); + if (active_chainstate.IsInitialBlockDownload()) return false; - if (::ChainActive().Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) + if (active_chainstate.m_chain.Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; - if (::ChainActive().Height() < pindexBestHeader->nHeight - 1) + if (active_chainstate.m_chain.Height() < pindexBestHeader->nHeight - 1) return false; return true; } @@ -364,10 +382,11 @@ static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) * and instead just erase from the mempool as needed. */ -static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) +static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) { AssertLockHeld(cs_main); AssertLockHeld(mempool.cs); + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); std::vector<uint256> vHashUpdate; // disconnectpool's insertion_order index sorts the entries from // oldest to newest, but the oldest entry will be the last tx from the @@ -378,10 +397,8 @@ static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransact auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions - TxValidationState stateDummy; if (!fAddToMempool || (*it)->IsCoinBase() || - !AcceptToMemoryPool(mempool, stateDummy, *it, - nullptr /* plTxnReplaced */, true /* bypass_limits */)) { + AcceptToMemoryPool(active_chainstate, mempool, *it, true /* bypass_limits */).m_result_type != MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); @@ -399,41 +416,47 @@ static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransact mempool.UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(&::ChainstateActive().CoinsTip(), ::ChainActive().Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + mempool.removeForReorg(active_chainstate, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions - LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + LimitMempoolSize(mempool, active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } -// Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool -// were somehow broken and returning the wrong scriptPubKeys -static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, - unsigned int flags, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +/** +* Checks to avoid mempool polluting consensus critical paths since cached +* signature and script validity results will be reused if we validate this +* transaction again during block validation. +* */ +static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, + const CCoinsViewCache& view, const CTxMemPool& pool, + unsigned int flags, PrecomputedTransactionData& txdata, CCoinsViewCache& coins_tip) + EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ AssertLockHeld(cs_main); - - // pool.cs should be locked already, but go ahead and re-take the lock here - // to enforce that mempool doesn't change between when we check the view - // and when we actually call through to CheckInputScripts - LOCK(pool.cs); + AssertLockHeld(pool.cs); assert(!tx.IsCoinBase()); for (const CTxIn& txin : tx.vin) { const Coin& coin = view.AccessCoin(txin.prevout); - // AcceptToMemoryPoolWorker has already checked that the coins are - // available, so this shouldn't fail. If the inputs are not available - // here then return false. + // This coin was checked in PreChecks and MemPoolAccept + // has been holding cs_main since then. + Assume(!coin.IsSpent()); if (coin.IsSpent()) return false; - // Check equivalence for available inputs. + // If the Coin is available, there are 2 possibilities: + // it is available in our current ChainstateActive UTXO set, + // or it's a UTXO provided by a transaction in our mempool. + // Ensure the scriptPubKeys in Coins from CoinsView are correct. const CTransactionRef& txFrom = pool.get(txin.prevout.hash); if (txFrom) { assert(txFrom->GetHash() == txin.prevout.hash); assert(txFrom->vout.size() > txin.prevout.n); assert(txFrom->vout[txin.prevout.n] == coin.out); } else { - const Coin& coinFromDisk = ::ChainstateActive().CoinsTip().AccessCoin(txin.prevout); - assert(!coinFromDisk.IsSpent()); - assert(coinFromDisk.out == coin.out); + assert(std::addressof(::ChainstateActive().CoinsTip()) == std::addressof(coins_tip)); + const Coin& coinFromUTXOSet = coins_tip.AccessCoin(txin.prevout); + assert(!coinFromUTXOSet.IsSpent()); + assert(coinFromUTXOSet.out == coin.out); } } @@ -446,19 +469,19 @@ namespace { class MemPoolAccept { public: - explicit MemPoolAccept(CTxMemPool& mempool) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&::ChainstateActive().CoinsTip(), m_pool), + explicit MemPoolAccept(CTxMemPool& mempool, CChainState& active_chainstate) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&active_chainstate.CoinsTip(), m_pool), m_active_chainstate(active_chainstate), m_limit_ancestors(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), m_limit_ancestor_size(gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), m_limit_descendants(gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), - m_limit_descendant_size(gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) {} + m_limit_descendant_size(gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) { + assert(std::addressof(::ChainstateActive()) == std::addressof(m_active_chainstate)); + } // We put the arguments we're handed into a struct, so we can pass them // around easier. struct ATMPArgs { const CChainParams& m_chainparams; - TxValidationState &m_state; const int64_t m_accept_time; - std::list<CTransactionRef>* m_replaced_transactions; const bool m_bypass_limits; /* * Return any outpoints which were not previously present in the coins @@ -469,11 +492,10 @@ public: */ std::vector<COutPoint>& m_coins_to_uncache; const bool m_test_accept; - CAmount* m_fee_out; }; // Single transaction acceptance - bool AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + MempoolAcceptResult AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); private: // All the intermediate state that gets passed between the various levels @@ -484,14 +506,17 @@ private: CTxMemPool::setEntries m_all_conflicting; CTxMemPool::setEntries m_ancestors; std::unique_ptr<CTxMemPoolEntry> m_entry; + std::list<CTransactionRef> m_replaced_transactions; bool m_replacement_transaction; + CAmount m_base_fees; CAmount m_modified_fees; CAmount m_conflicting_fees; size_t m_conflicting_size; const CTransactionRef& m_ptx; const uint256& m_hash; + TxValidationState m_state; }; // Run the policy checks on a given transaction, excluding any script checks. @@ -502,21 +527,21 @@ private: // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size // limiting is performed, false otherwise. - bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool Finalize(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. - bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) + bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) { CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { @@ -535,6 +560,8 @@ private: CCoinsViewMemPool m_viewmempool; CCoinsView m_dummy; + CChainState& m_active_chainstate; + // The package limits in effect at the time of invocation. const size_t m_limit_ancestors; const size_t m_limit_ancestor_size; @@ -551,12 +578,12 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256& hash = ws.m_hash; // Copy/alias what we need out of args - TxValidationState &state = args.m_state; const int64_t nAcceptTime = args.m_accept_time; const bool bypass_limits = args.m_bypass_limits; std::vector<COutPoint>& coins_to_uncache = args.m_coins_to_uncache; // Alias what we need out of ws + TxValidationState& state = ws.m_state; std::set<uint256>& setConflicts = ws.m_conflicts; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; CTxMemPool::setEntries& setAncestors = ws.m_ancestors; @@ -589,7 +616,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. - if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) + assert(std::addressof(::ChainActive()) == std::addressof(m_active_chainstate.m_chain)); + if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); // is it already in the memory pool? @@ -637,7 +665,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) LockPoints lp; m_view.SetBackend(m_viewmempool); - const CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); + assert(std::addressof(::ChainstateActive().CoinsTip()) == std::addressof(m_active_chainstate.CoinsTip())); + const CCoinsViewCache& coins_cache = m_active_chainstate.CoinsTip(); // do all inputs exist? for (const CTxIn& txin : tx.vin) { if (!coins_cache.HaveCoinInCache(txin.prevout)) { @@ -673,22 +702,19 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // be mined yet. // Must keep pool.cs for this unless we change CheckSequenceLocks to take a // CoinsViewCache instead of create its own - if (!CheckSequenceLocks(m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) + assert(std::addressof(::ChainstateActive()) == std::addressof(m_active_chainstate)); + if (!CheckSequenceLocks(m_active_chainstate, m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); - CAmount nFees = 0; - if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_active_chainstate.m_blockman)); + if (!Consensus::CheckTxInputs(tx, state, m_view, m_active_chainstate.m_blockman.GetSpendHeight(m_view), ws.m_base_fees)) { return false; // state filled in by CheckTxInputs } - // If fee_out is passed, return the fee to the caller - if (args.m_fee_out) { - *args.m_fee_out = nFees; - } - // Check for non-standard pay-to-script-hash in inputs const auto& params = args.m_chainparams.GetConsensus(); - auto taproot_state = VersionBitsState(::ChainActive().Tip(), params, Consensus::DEPLOYMENT_TAPROOT, versionbitscache); + assert(std::addressof(::ChainActive()) == std::addressof(m_active_chainstate.m_chain)); + auto taproot_state = VersionBitsState(m_active_chainstate.m_chain.Tip(), params, Consensus::DEPLOYMENT_TAPROOT, versionbitscache); if (fRequireStandard && !AreInputsStandard(tx, m_view, taproot_state == ThresholdState::ACTIVE)) { return state.Invalid(TxValidationResult::TX_INPUTS_NOT_STANDARD, "bad-txns-nonstandard-inputs"); } @@ -700,7 +726,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); // nModifiedFees includes any fee deltas from PrioritiseTransaction - nModifiedFees = nFees; + nModifiedFees = ws.m_base_fees; m_pool.ApplyDelta(hash, nModifiedFees); // Keep track of transactions that spend a coinbase, which we re-scan @@ -714,7 +740,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } - entry.reset(new CTxMemPoolEntry(ptx, nFees, nAcceptTime, ::ChainActive().Height(), + assert(std::addressof(::ChainActive()) == std::addressof(m_active_chainstate.m_chain)); + entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), fSpendsCoinbase, nSigOpsCost, lp)); unsigned int nSize = entry->GetTxSize(); @@ -918,11 +945,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; - - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; @@ -945,12 +971,11 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, Prec return true; } -bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; const CChainParams& chainparams = args.m_chainparams; // Check again against the current block tip's script verification @@ -968,8 +993,10 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, P // There is a similar check in CreateNewBlock() to prevent creating // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. - unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(::ChainActive().Tip(), chainparams.GetConsensus()); - if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata)) { + assert(std::addressof(::ChainActive()) == std::addressof(m_active_chainstate.m_chain)); + unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus()); + assert(std::addressof(::ChainstateActive().CoinsTip()) == std::addressof(m_active_chainstate.CoinsTip())); + if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata, m_active_chainstate.CoinsTip())) { return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", __func__, hash.ToString(), state.ToString()); } @@ -977,11 +1004,11 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, P return true; } -bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) +bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; const bool bypass_limits = args.m_bypass_limits; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; @@ -1000,8 +1027,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) hash.ToString(), FormatMoney(nModifiedFees - nConflictingFees), (int)entry->GetTxSize() - (int)nConflictingSize); - if (args.m_replaced_transactions) - args.m_replaced_transactions->push_back(it->GetSharedTx()); + ws.m_replaced_transactions.push_back(it->GetSharedTx()); } m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); @@ -1010,28 +1036,30 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) // - it's not being re-added during a reorg which bypasses typical mempool fee limits // - the node is not behind // - the transaction is not dependent on any other transactions in the mempool - bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation() && m_pool.HasNoInputsOf(tx); + assert(std::addressof(::ChainstateActive()) == std::addressof(m_active_chainstate)); + bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); // Store transaction in memory m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); // trim mempool and check if tx was trimmed if (!bypass_limits) { - LimitMempoolSize(m_pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + assert(std::addressof(::ChainstateActive().CoinsTip()) == std::addressof(m_active_chainstate.CoinsTip())); + LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); if (!m_pool.exists(hash)) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; } -bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) +MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) { AssertLockHeld(cs_main); LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) - Workspace workspace(ptx); + Workspace ws(ptx); - if (!PreChecks(args, workspace)) return false; + if (!PreChecks(args, ws)) return MempoolAcceptResult(ws.m_state); // Only compute the precomputed transaction data if we need to verify // scripts (ie, other policy checks pass). We perform the inexpensive @@ -1039,51 +1067,56 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs // checks pass, to mitigate CPU exhaustion denial-of-service attacks. PrecomputedTransactionData txdata; - if (!PolicyScriptChecks(args, workspace, txdata)) return false; + if (!PolicyScriptChecks(args, ws, txdata)) return MempoolAcceptResult(ws.m_state); - if (!ConsensusScriptChecks(args, workspace, txdata)) return false; + if (!ConsensusScriptChecks(args, ws, txdata)) return MempoolAcceptResult(ws.m_state); // Tx was accepted, but not added - if (args.m_test_accept) return true; + if (args.m_test_accept) { + return MempoolAcceptResult(std::move(ws.m_replaced_transactions), ws.m_base_fees); + } - if (!Finalize(args, workspace)) return false; + if (!Finalize(args, ws)) return MempoolAcceptResult(ws.m_state); GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); - return true; + return MempoolAcceptResult(std::move(ws.m_replaced_transactions), ws.m_base_fees); } } // anon namespace /** (try to) add transaction to memory pool with a specified acceptance time **/ -static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, + CChainState& active_chainstate, + const CTransactionRef &tx, int64_t nAcceptTime, + bool bypass_limits, bool test_accept) + EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, coins_to_uncache, test_accept, fee_out }; - bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); - if (!res) { + MemPoolAccept::ATMPArgs args { chainparams, nAcceptTime, bypass_limits, coins_to_uncache, test_accept }; + + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); + const MempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptSingleTransaction(tx, args); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { // Remove coins that were not present in the coins cache before calling ATMPW; // this is to prevent memory DoS in case we receive a large number of // invalid transactions that attempt to overrun the in-memory coins cache // (`CCoinsViewCache::cacheCoins`). for (const COutPoint& hashTx : coins_to_uncache) - ::ChainstateActive().CoinsTip().Uncache(hashTx); + active_chainstate.CoinsTip().Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits BlockValidationState state_dummy; - ::ChainstateActive().FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); - return res; + active_chainstate.FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); + return result; } -bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept, CAmount* fee_out) +MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPool& pool, const CTransactionRef& tx, + bool bypass_limits, bool test_accept) { - const CChainParams& chainparams = Params(); - return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, test_accept, fee_out); + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); + return AcceptToMemoryPoolWithTime(Params(), pool, active_chainstate, tx, GetTime(), bypass_limits, test_accept); } CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) @@ -1257,8 +1290,8 @@ void CoinsViews::InitCache() } CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash) - : m_blockman(blockman), - m_mempool(mempool), + : m_mempool(mempool), + m_blockman(blockman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1405,9 +1438,10 @@ bool CScriptCheck::operator()() { return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *txdata), &error); } -int GetSpendHeight(const CCoinsViewCache& inputs) +int BlockManager::GetSpendHeight(const CCoinsViewCache& inputs) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindexPrev = LookupBlockIndex(inputs.GetBestBlock()); return pindexPrev->nHeight + 1; } @@ -1451,7 +1485,10 @@ void InitScriptExecutionCache() { * * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp */ -bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, + const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, + bool cacheFullScriptStore, PrecomputedTransactionData& txdata, + std::vector<CScriptCheck>* pvChecks) { if (tx.IsCoinBase()) return true; @@ -1758,9 +1795,14 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationSt static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck(int worker_num) { - util::ThreadRename(strprintf("scriptch.%i", worker_num)); - scriptcheckqueue.Thread(); +void StartScriptCheckWorkerThreads(int threads_num) +{ + scriptcheckqueue.StartWorkerThreads(threads_num); +} + +void StopScriptCheckWorkerThreads() +{ + scriptcheckqueue.StopWorkerThreads(); } VersionBitsCache versionbitscache GUARDED_BY(cs_main); @@ -2236,17 +2278,25 @@ bool CChainState::FlushStateToDisk( { bool fFlushForPrune = false; bool fDoFullFlush = false; + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + // make sure we don't prune above the blockfilterindexes bestblocks + // pruning is height-based + int last_prune = m_chain.Height(); // last height we can prune + ForEachBlockFilterIndex([&](BlockFilterIndex& index) { + last_prune = std::max(1, std::min(last_prune, index.GetSummary().best_block_height)); + }); + if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); - m_blockman.FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight, m_chain.Height()); + m_blockman.FindFilesToPruneManual(setFilesToPrune, std::min(last_prune, nManualPruneHeight), m_chain.Height()); } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight(), m_chain.Height(), IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { @@ -2680,7 +2730,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai if (!DisconnectTip(state, chainparams, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. - UpdateMempoolForReorg(m_mempool, disconnectpool, false); + UpdateMempoolForReorg(::ChainstateActive(), m_mempool, disconnectpool, false); // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort @@ -2724,7 +2774,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai // A system error occurred (disk space, database error, ...). // Make the mempool consistent with the current tip, just in case // any observers try to use it before shutdown. - UpdateMempoolForReorg(m_mempool, disconnectpool, false); + UpdateMempoolForReorg(::ChainstateActive(), m_mempool, disconnectpool, false); return false; } } else { @@ -2741,7 +2791,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. - UpdateMempoolForReorg(m_mempool, disconnectpool, true); + UpdateMempoolForReorg(::ChainstateActive(), m_mempool, disconnectpool, true); } m_mempool.check(&CoinsTip()); @@ -2757,7 +2807,7 @@ static SynchronizationState GetSynchronizationState(bool init) return SynchronizationState::INIT_DOWNLOAD; } -static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { +static bool NotifyHeaderTip(CChainState& chainstate) LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex* pindexHeaderOld = nullptr; @@ -2768,7 +2818,8 @@ static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = ::ChainstateActive().IsInitialBlockDownload(); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + fInitialBlockDownload = chainstate.IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } @@ -2885,10 +2936,6 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar return true; } -bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { - return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); -} - bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { @@ -2981,7 +3028,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). - UpdateMempoolForReorg(m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + UpdateMempoolForReorg(::ChainstateActive(), m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); if (!ret) return false; assert(invalid_walk_tip->pprev == m_chain.Tip()); @@ -3193,7 +3240,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n bool finalize_undo = false; if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it @@ -3209,7 +3256,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n if ((int)nFile != nLastBlockFile) { if (!fKnown) { - LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; @@ -3388,14 +3435,14 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -//! Returns last CBlockIndex* that is a checkpoint -static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindex = LookupBlockIndex(hash); if (pindex) { return pindex; @@ -3413,7 +3460,7 @@ static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOC * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, BlockManager& blockman, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { assert(pindexPrev != nullptr); const int nHeight = pindexPrev->nHeight + 1; @@ -3428,7 +3475,8 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -3548,11 +3596,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator miSelf = m_block_index.find(hash); - CBlockIndex *pindex = nullptr; if (hash != chainparams.GetConsensus().hashGenesisBlock) { if (miSelf != m_block_index.end()) { // Block header is already known. - pindex = miSelf->second; + CBlockIndex* pindex = miSelf->second; if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { @@ -3579,7 +3626,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS LogPrintf("ERROR: %s: prev block invalid\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); /* Determine if this block descends from any block which has been found @@ -3621,8 +3668,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } } } - if (pindex == nullptr) - pindex = AddToBlockIndex(block); + CBlockIndex* pindex = AddToBlockIndex(block); if (ppindex) *ppindex = pindex; @@ -3633,6 +3679,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Exposed wrapper for AcceptBlockHeader bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); AssertLockNotHeld(cs_main); { LOCK(cs_main); @@ -3640,7 +3687,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); - ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); + ActiveChainstate().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { return false; @@ -3650,8 +3697,8 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } } } - if (NotifyHeaderTip()) { - if (::ChainstateActive().IsInitialBlockDownload() && ppindex && *ppindex) { + if (NotifyHeaderTip(ActiveChainstate())) { + if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", (*ppindex)->nHeight, 100.0/((*ppindex)->nHeight+(GetAdjustedTime() - (*ppindex)->GetBlockTime()) / Params().GetConsensus().nPowTargetSpacing) * (*ppindex)->nHeight); } } @@ -3763,6 +3810,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { AssertLockNotHeld(cs_main); + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); { CBlockIndex *pindex = nullptr; @@ -3778,7 +3826,7 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s bool ret = CheckBlock(*pblock, state, chainparams.GetConsensus()); if (ret) { // Store to disk - ret = ::ChainstateActive().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); + ret = ActiveChainstate().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } if (!ret) { GetMainSignals().BlockChecked(*pblock, state); @@ -3786,20 +3834,27 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s } } - NotifyHeaderTip(); + NotifyHeaderTip(ActiveChainstate()); BlockValidationState state; // Only used to report errors, not invalidity - ignore it - if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) + if (!ActiveChainstate().ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); return true; } -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW, + bool fCheckMerkleRoot) { AssertLockHeld(cs_main); - assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); - CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + assert(pindexPrev && pindexPrev == chainstate.m_chain.Tip()); + CCoinsViewCache viewNew(&chainstate.CoinsTip()); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -3807,13 +3862,14 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar indexDummy.phashBlock = &block_hash; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + assert(std::addressof(g_chainman.m_blockman) == std::addressof(chainstate.m_blockman)); + if (!ContextualCheckBlockHeader(block, state, chainstate.m_blockman, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, state.ToString()); if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -3915,7 +3971,7 @@ void PruneBlockFilesManual(int nManualPruneHeight) } } -void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, bool is_ibd) +void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd) { LOCK2(cs_main, cs_LastBlockFile); if (chain_tip_height < 0 || nPruneTarget == 0) { @@ -3925,7 +3981,7 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr return; } - unsigned int nLastBlockWeCanPrune = chain_tip_height - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = std::min(prune_height, chain_tip_height - static_cast<int>(MIN_BLOCKS_TO_KEEP)); uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -3976,7 +4032,7 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr static FlatFileSeq BlockFileSeq() { - return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); + return FlatFileSeq(GetBlocksDir(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); } static FlatFileSeq UndoFileSeq() @@ -4142,7 +4198,8 @@ bool static LoadBlockIndexDB(ChainstateManager& chainman, const CChainParams& ch void CChainState::LoadMempool(const ArgsManager& args) { if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(m_mempool); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + ::LoadMempool(m_mempool, *this); } m_mempool.SetIsLoaded(!ShutdownRequested()); } @@ -4159,7 +4216,7 @@ bool CChainState::LoadChainTip(const CChainParams& chainparams) } // Load pointer to end of best chain - CBlockIndex* pindex = LookupBlockIndex(coins_cache.GetBestBlock()); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(coins_cache.GetBestBlock()); if (!pindex) { return false; } @@ -4589,7 +4646,7 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return ::ChainstateActive().LoadGenesisBlock(chainparams); } -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) +void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4638,7 +4695,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(block.hashPrevBlock)) { + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + if (hash != chainparams.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) @@ -4647,10 +4705,12 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } // process in case the block isn't known yet - CBlockIndex* pindex = LookupBlockIndex(hash); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; - if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { nLoaded++; } if (state.IsError()) { @@ -4664,12 +4724,14 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); // Recursively process earlier encountered successors of this block std::deque<uint256> queue; @@ -4687,7 +4749,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi head.ToString()); LOCK(cs_main); BlockValidationState dummy; - if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -4695,7 +4758,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } range.first++; mapBlocksUnknownParent.erase(it); - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); } } } catch (const std::exception& e) { @@ -4965,7 +5029,7 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool(CTxMemPool& pool) +bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate) { const CChainParams& chainparams = Params(); int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; @@ -5003,13 +5067,11 @@ bool LoadMempool(CTxMemPool& pool) if (amountdelta) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } - TxValidationState state; if (nTime > nNow - nExpiryTimeout) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nTime, - nullptr /* plTxnReplaced */, false /* bypass_limits */, - false /* test_accept */); - if (state.IsValid()) { + assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); + if (AcceptToMemoryPoolWithTime(chainparams, pool, active_chainstate, tx, nTime, false /* bypass_limits */, + false /* test_accept */).m_result_type == MempoolAcceptResult::ResultType::VALID) { ++count; } else { // mempool may contain the transaction already, e.g. from @@ -5133,7 +5195,9 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin } Optional<uint256> ChainstateManager::SnapshotBlockhash() const { - if (m_active_chainstate != nullptr) { + LOCK(::cs_main); + if (m_active_chainstate != nullptr && + !m_active_chainstate->m_from_snapshot_blockhash.IsNull()) { // If a snapshot chainstate exists, it will always be our active. return m_active_chainstate->m_from_snapshot_blockhash; } @@ -5142,6 +5206,7 @@ Optional<uint256> ChainstateManager::SnapshotBlockhash() const { std::vector<CChainState*> ChainstateManager::GetAll() { + LOCK(::cs_main); std::vector<CChainState*> out; if (!IsSnapshotValidated() && m_ibd_chainstate) { @@ -5177,19 +5242,311 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const return *to_modify; } +const AssumeutxoData* ExpectedAssumeutxo( + const int height, const CChainParams& chainparams) +{ + const MapAssumeutxo& valid_assumeutxos_map = chainparams.Assumeutxo(); + const auto assumeutxo_found = valid_assumeutxos_map.find(height); + + if (assumeutxo_found != valid_assumeutxos_map.end()) { + return &assumeutxo_found->second; + } + return nullptr; +} + +bool ChainstateManager::ActivateSnapshot( + CAutoFile& coins_file, + const SnapshotMetadata& metadata, + bool in_memory) +{ + uint256 base_blockhash = metadata.m_base_blockhash; + + if (this->SnapshotBlockhash()) { + LogPrintf("[snapshot] can't activate a snapshot-based chainstate more than once\n"); + return false; + } + + int64_t current_coinsdb_cache_size{0}; + int64_t current_coinstip_cache_size{0}; + + // Cache percentages to allocate to each chainstate. + // + // These particular percentages don't matter so much since they will only be + // relevant during snapshot activation; caches are rebalanced at the conclusion of + // this function. We want to give (essentially) all available cache capacity to the + // snapshot to aid the bulk load later in this function. + static constexpr double IBD_CACHE_PERC = 0.01; + static constexpr double SNAPSHOT_CACHE_PERC = 0.99; + + { + LOCK(::cs_main); + // Resize the coins caches to ensure we're not exceeding memory limits. + // + // Allocate the majority of the cache to the incoming snapshot chainstate, since + // (optimistically) getting to its tip will be the top priority. We'll need to call + // `MaybeRebalanceCaches()` once we're done with this function to ensure + // the right allocation (including the possibility that no snapshot was activated + // and that we should restore the active chainstate caches to their original size). + // + current_coinsdb_cache_size = this->ActiveChainstate().m_coinsdb_cache_size_bytes; + current_coinstip_cache_size = this->ActiveChainstate().m_coinstip_cache_size_bytes; + + // Temporarily resize the active coins cache to make room for the newly-created + // snapshot chain. + this->ActiveChainstate().ResizeCoinsCaches( + static_cast<size_t>(current_coinstip_cache_size * IBD_CACHE_PERC), + static_cast<size_t>(current_coinsdb_cache_size * IBD_CACHE_PERC)); + } + + auto snapshot_chainstate = WITH_LOCK(::cs_main, return MakeUnique<CChainState>( + this->ActiveChainstate().m_mempool, m_blockman, base_blockhash)); + + { + LOCK(::cs_main); + snapshot_chainstate->InitCoinsDB( + static_cast<size_t>(current_coinsdb_cache_size * SNAPSHOT_CACHE_PERC), + in_memory, false, "chainstate"); + snapshot_chainstate->InitCoinsCache( + static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); + } + + const bool snapshot_ok = this->PopulateAndValidateSnapshot( + *snapshot_chainstate, coins_file, metadata); + + if (!snapshot_ok) { + WITH_LOCK(::cs_main, this->MaybeRebalanceCaches()); + return false; + } + + { + LOCK(::cs_main); + assert(!m_snapshot_chainstate); + m_snapshot_chainstate.swap(snapshot_chainstate); + const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(::Params()); + assert(chaintip_loaded); + + m_active_chainstate = m_snapshot_chainstate.get(); + + LogPrintf("[snapshot] successfully activated snapshot %s\n", base_blockhash.ToString()); + LogPrintf("[snapshot] (%.2f MB)\n", + m_snapshot_chainstate->CoinsTip().DynamicMemoryUsage() / (1000 * 1000)); + + this->MaybeRebalanceCaches(); + } + return true; +} + +bool ChainstateManager::PopulateAndValidateSnapshot( + CChainState& snapshot_chainstate, + CAutoFile& coins_file, + const SnapshotMetadata& metadata) +{ + // It's okay to release cs_main before we're done using `coins_cache` because we know + // that nothing else will be referencing the newly created snapshot_chainstate yet. + CCoinsViewCache& coins_cache = *WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsTip()); + + uint256 base_blockhash = metadata.m_base_blockhash; + + COutPoint outpoint; + Coin coin; + const uint64_t coins_count = metadata.m_coins_count; + uint64_t coins_left = metadata.m_coins_count; + + LogPrintf("[snapshot] loading coins from snapshot %s\n", base_blockhash.ToString()); + int64_t flush_now{0}; + int64_t coins_processed{0}; + + while (coins_left > 0) { + try { + coins_file >> outpoint; + } catch (const std::ios_base::failure&) { + LogPrintf("[snapshot] bad snapshot - no coins left after deserializing %d coins\n", + coins_count - coins_left); + return false; + } + coins_file >> coin; + coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + + --coins_left; + ++coins_processed; + + if (coins_processed % 1000000 == 0) { + LogPrintf("[snapshot] %d coins loaded (%.2f%%, %.2f MB)\n", + coins_processed, + static_cast<float>(coins_processed) * 100 / static_cast<float>(coins_count), + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + } + + // Batch write and flush (if we need to) every so often. + // + // If our average Coin size is roughly 41 bytes, checking every 120,000 coins + // means <5MB of memory imprecision. + if (coins_processed % 120000 == 0) { + if (ShutdownRequested()) { + return false; + } + + const auto snapshot_cache_state = WITH_LOCK(::cs_main, + return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool)); + + if (snapshot_cache_state >= + CoinsCacheSizeState::CRITICAL) { + LogPrintf("[snapshot] flushing coins cache (%.2f MB)... ", /* Continued */ + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + flush_now = GetTimeMillis(); + + // This is a hack - we don't know what the actual best block is, but that + // doesn't matter for the purposes of flushing the cache here. We'll set this + // to its correct value (`base_blockhash`) below after the coins are loaded. + coins_cache.SetBestBlock(GetRandHash()); + + coins_cache.Flush(); + LogPrintf("done (%.2fms)\n", GetTimeMillis() - flush_now); + } + } + } + + // Important that we set this. This and the coins_cache accesses above are + // sort of a layer violation, but either we reach into the innards of + // CCoinsViewCache here or we have to invert some of the CChainState to + // embed them in a snapshot-activation-specific CCoinsViewCache bulk load + // method. + coins_cache.SetBestBlock(base_blockhash); + + bool out_of_coins{false}; + try { + coins_file >> outpoint; + } catch (const std::ios_base::failure&) { + // We expect an exception since we should be out of coins. + out_of_coins = true; + } + if (!out_of_coins) { + LogPrintf("[snapshot] bad snapshot - coins left over after deserializing %d coins\n", + coins_count); + return false; + } + + LogPrintf("[snapshot] loaded %d (%.2f MB) coins from snapshot %s\n", + coins_count, + coins_cache.DynamicMemoryUsage() / (1000 * 1000), + base_blockhash.ToString()); + + LogPrintf("[snapshot] flushing snapshot chainstate to disk\n"); + // No need to acquire cs_main since this chainstate isn't being used yet. + coins_cache.Flush(); // TODO: if #17487 is merged, add erase=false here for better performance. + + assert(coins_cache.GetBestBlock() == base_blockhash); + + CCoinsStats stats; + auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; + + // As above, okay to immediately release cs_main here since no other context knows + // about the snapshot_chainstate. + CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); + + if (!GetUTXOStats(snapshot_coinsdb, stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) { + LogPrintf("[snapshot] failed to generate coins stats\n"); + return false; + } + + // Ensure that the base blockhash appears in the known chain of valid headers. We're willing to + // wait a bit here because the snapshot may have been loaded on startup, before we've + // received headers from the network. + + int max_secs_to_wait_for_headers = 60 * 10; + CBlockIndex* snapshot_start_block = nullptr; + + while (max_secs_to_wait_for_headers > 0) { + snapshot_start_block = WITH_LOCK(::cs_main, + return m_blockman.LookupBlockIndex(base_blockhash)); + --max_secs_to_wait_for_headers; + + if (!snapshot_start_block) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } else { + break; + } + } + + if (snapshot_start_block == nullptr) { + LogPrintf("[snapshot] timed out waiting for snapshot start blockheader %s\n", + base_blockhash.ToString()); + return false; + } + + // Assert that the deserialized chainstate contents match the expected assumeutxo value. + + int base_height = snapshot_start_block->nHeight; + auto maybe_au_data = ExpectedAssumeutxo(base_height, ::Params()); + + if (!maybe_au_data) { + LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " /* Continued */ + "(%d) - refusing to load snapshot\n", base_height); + return false; + } + + const AssumeutxoData& au_data = *maybe_au_data; + + if (stats.hashSerialized != au_data.hash_serialized) { + LogPrintf("[snapshot] bad snapshot content hash: expected %s, got %s\n", + au_data.hash_serialized.ToString(), stats.hashSerialized.ToString()); + return false; + } + + snapshot_chainstate.m_chain.SetTip(snapshot_start_block); + + // The remainder of this function requires modifying data protected by cs_main. + LOCK(::cs_main); + + // Fake various pieces of CBlockIndex state: + // + // - nChainTx: so that we accurately report IBD-to-tip progress + // - nTx: so that LoadBlockIndex() loads assumed-valid CBlockIndex entries + // (among other things) + // - nStatus & BLOCK_OPT_WITNESS: so that RewindBlockIndex() doesn't zealously + // unwind the assumed-valid chain. + // + CBlockIndex* index = nullptr; + for (int i = 0; i <= snapshot_chainstate.m_chain.Height(); ++i) { + index = snapshot_chainstate.m_chain[i]; + + if (!index->nTx) { + index->nTx = 1; + } + index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + + // We need to fake this flag so that CChainState::RewindBlockIndex() + // won't try to rewind the entire assumed-valid chain on startup. + if (index->pprev && ::IsWitnessEnabled(index->pprev, ::Params().GetConsensus())) { + index->nStatus |= BLOCK_OPT_WITNESS; + } + } + + assert(index); + index->nChainTx = metadata.m_nchaintx; + snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block); + + LogPrintf("[snapshot] validated snapshot (%.2f MB)\n", + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + return true; +} + CChainState& ChainstateManager::ActiveChainstate() const { + LOCK(::cs_main); assert(m_active_chainstate); return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const { + LOCK(::cs_main); return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } CChainState& ChainstateManager::ValidatedChainstate() const { + LOCK(::cs_main); if (m_snapshot_chainstate && IsSnapshotValidated()) { return *m_snapshot_chainstate.get(); } @@ -5199,6 +5556,7 @@ CChainState& ChainstateManager::ValidatedChainstate() const bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const { + LOCK(::cs_main); return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); } @@ -5214,6 +5572,7 @@ void ChainstateManager::Unload() void ChainstateManager::Reset() { + LOCK(::cs_main); m_ibd_chainstate.reset(); m_snapshot_chainstate.reset(); m_active_chainstate = nullptr; diff --git a/src/validation.h b/src/validation.h index d10b260d8a..4e4bdbea54 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,9 +11,12 @@ #endif #include <amount.h> +#include <attributes.h> #include <coins.h> +#include <consensus/validation.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <node/utxo_snapshot.h> #include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars @@ -23,6 +26,8 @@ #include <txdb.h> #include <versionbits.h> #include <serialize.h> +#include <util/check.h> +#include <util/hasher.h> #include <atomic> #include <map> @@ -39,17 +44,18 @@ class CBlockIndex; class CBlockTreeDB; class CBlockUndo; class CChainParams; +struct CCheckpointData; class CInv; class CConnman; class CScriptCheck; class CTxMemPool; class ChainstateManager; -class TxValidationState; struct ChainTxData; struct DisconnectedBlockTransactions; struct PrecomputedTransactionData; struct LockPoints; +struct AssumeutxoData; /** Default for -minrelaytxfee, minimum relay fee for transactions */ static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; @@ -93,14 +99,6 @@ static const unsigned int DEFAULT_CHECKLEVEL = 3; // Setting the target to >= 550 MiB will make it likely we can respect the target. static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; -struct BlockHasher -{ - // this used to call `GetCheapHash()` in uint256, which was later moved; the - // cheap hash function simply calls ReadLE64() however, so the end result is - // identical - size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } -}; - /** Current sync state passed to tip changed callbacks. */ enum class SynchronizationState { INIT_REINDEX, @@ -150,14 +148,14 @@ extern const std::vector<std::string> CHECKLEVEL_DOC; FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos &pos); -/** Import blocks from an external file */ -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); -/** Run an instance of the script checking thread */ -void ThreadScriptCheck(int worker_num); +/** Run instances of script checking worker threads */ +void StartScriptCheckWorkerThreads(int threads_num); +/** Stop all of the script checking worker threads */ +void StopScriptCheckWorkerThreads(); /** * Return transaction from the block at block_index. * If block_index is not provided, fall back to mempool. @@ -171,13 +169,6 @@ void ThreadScriptCheck(int worker_num); * @returns The tx if found, otherwise nullptr */ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); -/** - * Find the best known block, and make it the tip of the block chain - * - * May not be called with cs_main held. May not be called in a - * validationinterface callback. - */ -bool ActivateBestChain(BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ @@ -194,12 +185,46 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); /** Prune block files up to a given height */ void PruneBlockFilesManual(int nManualPruneHeight); -/** (try to) add transaction to memory pool - * plTxnReplaced will be appended to with all transactions replaced from mempool - * @param[out] fee_out optional argument to return tx fee to the caller **/ -bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept=false, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** +* Validation result for a single transaction mempool acceptance. +*/ +struct MempoolAcceptResult { + /** Used to indicate the results of mempool validation, + * including the possibility of unfinished validation. + */ + enum class ResultType { + VALID, //!> Fully validated, valid. + INVALID, //!> Invalid. + }; + ResultType m_result_type; + TxValidationState m_state; + + // The following fields are only present when m_result_type = ResultType::VALID + /** Mempool transactions replaced by the tx per BIP 125 rules. */ + std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Raw base fees. */ + std::optional<CAmount> m_base_fees; + + /** Constructor for failure case */ + explicit MempoolAcceptResult(TxValidationState state) + : m_result_type(ResultType::INVALID), + m_state(state), m_replaced_transactions(nullopt), m_base_fees(nullopt) { + Assume(!state.IsValid()); // Can be invalid or error + } + + /** Constructor for success case */ + explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) + : m_result_type(ResultType::VALID), m_state(TxValidationState{}), + m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} +}; + +/** + * (Try to) add a transaction to the memory pool. + * @param[in] bypass_limits When true, don't enforce mempool fee limits. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + */ +MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPool& pool, const CTransactionRef& tx, + bool bypass_limits, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); @@ -223,12 +248,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); * * See consensus/consensus.h for flag definitions. */ -bool CheckFinalTx(const CTransaction &tx, int flags = -1) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags = -1) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Test whether the LockPoints height and time are still valid on the current chain */ -bool TestLockPointValidity(const LockPoints* lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Check if transaction will be BIP 68 final in the next block to be created. @@ -241,7 +266,12 @@ bool TestLockPointValidity(const LockPoints* lp) EXCLUSIVE_LOCKS_REQUIRED(cs_mai * * See consensus/consensus.h for flag definitions. */ -bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flags, LockPoints* lp = nullptr, bool useExistingLockPoints = false) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs); +bool CheckSequenceLocks(CChainState& active_chainstate, + const CTxMemPool& pool, + const CTransaction& tx, + int flags, + LockPoints* lp = nullptr, + bool useExistingLockPoints = false) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs); /** * Closure representing one script verification @@ -296,7 +326,13 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Check a block is completely valid from start to finish (only works on top of our current best block) */ -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW = true, + bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ @@ -316,11 +352,6 @@ public: bool VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth); }; -CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** Find the last common block between the parameter chain and a locator. */ -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - enum DisconnectResult { DISCONNECT_OK, // All good. @@ -373,7 +404,7 @@ private: * * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned */ - void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, bool is_ibd); + void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd); public: BlockMap m_block_index GUARDED_BY(cs_main); @@ -438,6 +469,21 @@ public: const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Find the last common block between the parameter chain and a locator. */ + CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Returns last CBlockIndex* that is a checkpoint + CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * Return the spend height, which is one more than the inputs.GetBestBlock(). + * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) + * This is also true for mempool checks. + */ + int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + ~BlockManager() { Unload(); } @@ -530,11 +576,6 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! Reference to a BlockManager instance which itself is shared across all - //! CChainState instances. Keeping a local reference allows us to test more - //! easily as opposed to referencing a global. - BlockManager& m_blockman; - //! mempool that is kept in sync with the chain CTxMemPool& m_mempool; @@ -542,6 +583,10 @@ protected: std::unique_ptr<CoinsViews> m_coins_views; public: + //! Reference to a BlockManager instance which itself is shared across all + //! CChainState instances. + BlockManager& m_blockman; + explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** @@ -618,6 +663,9 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Import blocks from an external file */ + void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -643,9 +691,10 @@ public: void PruneAndFlush(); /** - * Make the best chain active, in multiple steps. The result is either failure - * or an activated best chain. pblock is either nullptr or a pointer to a block - * that is already loaded (to avoid loading it again from disk). + * Find the best known block, and make it the tip of the block chain. The + * result is either failure or an activated best chain. pblock is either + * nullptr or a pointer to a block that is already loaded (to avoid loading + * it again from disk). * * ActivateBestChain is split into steps (see ActivateBestChainStep) so that * we avoid holding cs_main for an extended period of time; the length of this @@ -659,7 +708,7 @@ public: bool ActivateBestChain( BlockValidationState& state, const CChainParams& chainparams, - std::shared_ptr<const CBlock> pblock) LOCKS_EXCLUDED(cs_main); + std::shared_ptr<const CBlock> pblock = nullptr) LOCKS_EXCLUDED(cs_main); bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -789,45 +838,45 @@ private: //! using this pointer (e.g. net_processing). //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_ibd_chainstate; + std::unique_ptr<CChainState> m_ibd_chainstate GUARDED_BY(::cs_main); //! A chainstate initialized on the basis of a UTXO snapshot. If this is //! non-null, it is always our active chainstate. //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_snapshot_chainstate; + std::unique_ptr<CChainState> m_snapshot_chainstate GUARDED_BY(::cs_main); //! Points to either the ibd or snapshot chainstate; indicates our //! most-work chain. //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - CChainState* m_active_chainstate{nullptr}; + CChainState* m_active_chainstate GUARDED_BY(::cs_main) {nullptr}; //! If true, the assumed-valid chainstate has been fully validated //! by the background validation chainstate. bool m_snapshot_validated{false}; + //! Internal helper for ActivateSnapshot(). + [[nodiscard]] bool PopulateAndValidateSnapshot( + CChainState& snapshot_chainstate, + CAutoFile& coins_file, + const SnapshotMetadata& metadata); + // For access to m_active_chainstate. friend CChainState& ChainstateActive(); friend CChain& ChainActive(); @@ -858,6 +907,22 @@ public: //! Get all chainstates currently being used. std::vector<CChainState*> GetAll(); + //! Construct and activate a Chainstate on the basis of UTXO snapshot data. + //! + //! Steps: + //! + //! - Initialize an unused CChainState. + //! - Load its `CoinsViews` contents from `coins_file`. + //! - Verify that the hash of the resulting coinsdb matches the expected hash + //! per assumeutxo chain parameters. + //! - Wait for our headers chain to include the base block of the snapshot. + //! - "Fast forward" the tip of the new chainstate to the base of the snapshot, + //! faking nTx* block index data along the way. + //! - Move the new chainstate to `m_snapshot_chainstate` and make it our + //! ChainstateActive(). + [[nodiscard]] bool ActivateSnapshot( + CAutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory); + //! The most-work chain. CChainState& ActiveChainstate() const; CChain& ActiveChain() const { return ActiveChainstate().m_chain; } @@ -950,13 +1015,6 @@ CChain& ChainActive(); /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; -/** - * Return the spend height, which is one more than the inputs.GetBestBlock(). - * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) - * This is also true for mempool checks. - */ -int GetSpendHeight(const CCoinsViewCache& inputs); - extern VersionBitsCache versionbitscache; /** @@ -971,7 +1029,7 @@ CBlockFileInfo* GetBlockFileInfo(size_t n); bool DumpMempool(const CTxMemPool& pool); /** Load the mempool from disk. */ -bool LoadMempool(CTxMemPool& pool); +bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate); //! Check whether the block associated with this index entry is pruned or not. inline bool IsBlockPruned(const CBlockIndex* pblockindex) @@ -979,4 +1037,13 @@ inline bool IsBlockPruned(const CBlockIndex* pblockindex) return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); } +/** + * Return the expected assumeutxo value for a given height, if one exists. + * + * @param height[in] Get the assumeutxo value for this height. + * + * @returns empty if no assumeutxo configuration exists for the given height. + */ +const AssumeutxoData* ExpectedAssumeutxo(const int height, const CChainParams& params); + #endif // BITCOIN_VALIDATION_H diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 6ed48593fb..ad40e6da9a 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -488,9 +488,9 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) break; } if (pszSkip && - strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) + strncmp((const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) continue; - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { + if (strncmp((const char*)ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; @@ -723,6 +723,23 @@ bool BerkeleyBatch::TxnAbort() return (ret == 0); } +bool BerkeleyDatabaseSanityCheck() +{ + int major, minor; + DbEnv::version(&major, &minor, nullptr); + + /* If the major version differs, or the minor version of library is *older* + * than the header that was compiled against, flag an error. + */ + if (major != DB_VERSION_MAJOR || minor < DB_VERSION_MINOR) { + LogPrintf("BerkeleyDB database version conflict: header version is %d.%d, library version is %d.%d\n", + DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor); + return false; + } + + return true; +} + std::string BerkeleyDatabaseVersion() { return DbEnv::version(nullptr, nullptr, nullptr); diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index bf1617d67f..a8209587d7 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -223,6 +223,10 @@ public: std::string BerkeleyDatabaseVersion(); +/** Perform sanity check of runtime BDB version versus linked BDB version. + */ +bool BerkeleyDatabaseSanityCheck(); + //! Return object giving access to Berkeley database at specified path. std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index a2dea84d17..10f89e3a6f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -300,8 +300,26 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group ******************************************************************************/ -void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) { +void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only) { + // Compute the effective value first + const CAmount coin_fee = output.m_input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.m_input_bytes); + const CAmount ev = output.txout.nValue - coin_fee; + + // Filter for positive only here before adding the coin + if (positive_only && ev <= 0) return; + m_outputs.push_back(output); + CInputCoin& coin = m_outputs.back(); + + coin.m_fee = coin_fee; + fee += coin.m_fee; + + coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.m_input_bytes); + long_term_fee += coin.m_long_term_fee; + + coin.effective_value = ev; + effective_value += coin.effective_value; + m_from_me &= from_me; m_value += output.txout.nValue; m_depth = std::min(m_depth, depth); @@ -312,20 +330,6 @@ void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size // descendants is the count as seen from the top ancestor, not the descendants as seen from the // coin itself; thus, this value is counted as the max, not the sum m_descendants = std::max(m_descendants, descendants); - effective_value += output.effective_value; - fee += output.m_fee; - long_term_fee += output.m_long_term_fee; -} - -std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) { - auto it = m_outputs.begin(); - while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it; - if (it == m_outputs.end()) return it; - m_value -= output.txout.nValue; - effective_value -= output.effective_value; - fee -= output.m_fee; - long_term_fee -= output.m_long_term_fee; - return m_outputs.erase(it); } bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const @@ -334,35 +338,3 @@ bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_f && m_ancestors <= eligibility_filter.max_ancestors && m_descendants <= eligibility_filter.max_descendants; } - -void OutputGroup::SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate) -{ - fee = 0; - long_term_fee = 0; - effective_value = 0; - for (CInputCoin& coin : m_outputs) { - coin.m_fee = coin.m_input_bytes < 0 ? 0 : effective_feerate.GetFee(coin.m_input_bytes); - fee += coin.m_fee; - - coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); - long_term_fee += coin.m_long_term_fee; - - coin.effective_value = coin.txout.nValue - coin.m_fee; - effective_value += coin.effective_value; - } -} - -OutputGroup OutputGroup::GetPositiveOnlyGroup() -{ - OutputGroup group(*this); - for (auto it = group.m_outputs.begin(); it != group.m_outputs.end(); ) { - const CInputCoin& coin = *it; - // Only include outputs that are positive effective value (i.e. not dust) - if (coin.effective_value <= 0) { - it = group.Discard(coin); - } else { - ++it; - } - } - return group; -} diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index cf9768b927..f0e1addaf1 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -6,11 +6,10 @@ #define BITCOIN_WALLET_COINSELECTION_H #include <amount.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -class CFeeRate; - //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -63,9 +62,11 @@ struct CoinEligibilityFilter const int conf_theirs; const uint64_t max_ancestors; const uint64_t max_descendants; + const bool m_include_partial_groups{false}; //! Include partial destination groups when avoid_reuse and there are full groups CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} + CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants, bool include_partial) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants), m_include_partial_groups(include_partial) {} }; struct OutputGroup @@ -78,27 +79,18 @@ struct OutputGroup size_t m_descendants{0}; CAmount effective_value{0}; CAmount fee{0}; + CFeeRate m_effective_feerate{0}; CAmount long_term_fee{0}; + CFeeRate m_long_term_feerate{0}; OutputGroup() {} - OutputGroup(std::vector<CInputCoin>&& outputs, bool from_me, CAmount value, int depth, size_t ancestors, size_t descendants) - : m_outputs(std::move(outputs)) - , m_from_me(from_me) - , m_value(value) - , m_depth(depth) - , m_ancestors(ancestors) - , m_descendants(descendants) + OutputGroup(const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate) : + m_effective_feerate(effective_feerate), + m_long_term_feerate(long_term_feerate) {} - OutputGroup(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) : OutputGroup() { - Insert(output, depth, from_me, ancestors, descendants); - } - void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants); - std::vector<CInputCoin>::iterator Discard(const CInputCoin& output); - bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; - //! Update the OutputGroup's fee, long_term_fee, and effective_value based on the given feerates - void SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate); - OutputGroup GetPositiveOnlyGroup(); + void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only); + bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; }; bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees); diff --git a/src/wallet/external_signer.cpp b/src/wallet/external_signer.cpp new file mode 100644 index 0000000000..3396111760 --- /dev/null +++ b/src/wallet/external_signer.cpp @@ -0,0 +1,119 @@ +// Copyright (c) 2018-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <core_io.h> +#include <psbt.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <wallet/external_signer.h> + +ExternalSigner::ExternalSigner(const std::string& command, const std::string& fingerprint, std::string chain, std::string name): m_command(command), m_fingerprint(fingerprint), m_chain(chain), m_name(name) {} + +const std::string ExternalSigner::NetworkArg() const +{ + return " --chain " + m_chain; +} + +#ifdef ENABLE_EXTERNAL_SIGNER + +bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalSigner>& signers, std::string chain, bool ignore_errors) +{ + // Call <command> enumerate + const UniValue result = RunCommandParseJSON(command + " enumerate"); + if (!result.isArray()) { + if (ignore_errors) return false; + throw ExternalSignerException(strprintf("'%s' received invalid response, expected array of signers", command)); + } + for (UniValue signer : result.getValues()) { + // Check for error + const UniValue& error = find_value(signer, "error"); + if (!error.isNull()) { + if (ignore_errors) return false; + if (!error.isStr()) { + throw ExternalSignerException(strprintf("'%s' error", command)); + } + throw ExternalSignerException(strprintf("'%s' error: %s", command, error.getValStr())); + } + // Check if fingerprint is present + const UniValue& fingerprint = find_value(signer, "fingerprint"); + if (fingerprint.isNull()) { + if (ignore_errors) return false; + throw ExternalSignerException(strprintf("'%s' received invalid response, missing signer fingerprint", command)); + } + std::string fingerprintStr = fingerprint.get_str(); + // Skip duplicate signer + bool duplicate = false; + for (ExternalSigner signer : signers) { + if (signer.m_fingerprint.compare(fingerprintStr) == 0) duplicate = true; + } + if (duplicate) break; + std::string name = ""; + const UniValue& model_field = find_value(signer, "model"); + if (model_field.isStr() && model_field.getValStr() != "") { + name += model_field.getValStr(); + } + signers.push_back(ExternalSigner(command, fingerprintStr, chain, name)); + } + return true; +} + +UniValue ExternalSigner::DisplayAddress(const std::string& descriptor) const +{ + return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " displayaddress --desc \"" + descriptor + "\""); +} + +UniValue ExternalSigner::GetDescriptors(int account) +{ + return RunCommandParseJSON(m_command + " --fingerprint \"" + m_fingerprint + "\"" + NetworkArg() + " getdescriptors --account " + strprintf("%d", account)); +} + +bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::string& error) +{ + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + + // Check if signer fingerprint matches any input master key fingerprint + bool match = false; + for (unsigned int i = 0; i < psbtx.inputs.size(); ++i) { + const PSBTInput& input = psbtx.inputs[i]; + for (auto entry : input.hd_keypaths) { + if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) match = true; + } + } + + if (!match) { + error = "Signer fingerprint " + m_fingerprint + " does not match any of the inputs:\n" + EncodeBase64(ssTx.str()); + return false; + } + + std::string command = m_command + " --stdin --fingerprint \"" + m_fingerprint + "\"" + NetworkArg(); + std::string stdinStr = "signtx \"" + EncodeBase64(ssTx.str()) + "\""; + + const UniValue signer_result = RunCommandParseJSON(command, stdinStr); + + if (find_value(signer_result, "error").isStr()) { + error = find_value(signer_result, "error").get_str(); + return false; + } + + if (!find_value(signer_result, "psbt").isStr()) { + error = "Unexpected result from signer"; + return false; + } + + PartiallySignedTransaction signer_psbtx; + std::string signer_psbt_error; + if (!DecodeBase64PSBT(signer_psbtx, find_value(signer_result, "psbt").get_str(), signer_psbt_error)) { + error = strprintf("TX decode failed %s", signer_psbt_error); + return false; + } + + psbtx = signer_psbtx; + + return true; +} + +#endif diff --git a/src/wallet/external_signer.h b/src/wallet/external_signer.h new file mode 100644 index 0000000000..4b9711107b --- /dev/null +++ b/src/wallet/external_signer.h @@ -0,0 +1,73 @@ +// Copyright (c) 2018-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_EXTERNAL_SIGNER_H +#define BITCOIN_WALLET_EXTERNAL_SIGNER_H + +#include <stdexcept> +#include <string> +#include <univalue.h> +#include <util/system.h> + +struct PartiallySignedTransaction; + +class ExternalSignerException : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +//! Enables interaction with an external signing device or service, such as +//! a hardware wallet. See doc/external-signer.md +class ExternalSigner +{ +private: + //! The command which handles interaction with the external signer. + std::string m_command; + +public: + //! @param[in] command the command which handles interaction with the external signer + //! @param[in] fingerprint master key fingerprint of the signer + //! @param[in] chain "main", "test", "regtest" or "signet" + //! @param[in] name device name + ExternalSigner(const std::string& command, const std::string& fingerprint, std::string chain, std::string name); + + //! Master key fingerprint of the signer + std::string m_fingerprint; + + //! Bitcoin mainnet, testnet, etc + std::string m_chain; + + //! Name of signer + std::string m_name; + + const std::string NetworkArg() const; + +#ifdef ENABLE_EXTERNAL_SIGNER + //! Obtain a list of signers. Calls `<command> enumerate`. + //! @param[in] command the command which handles interaction with the external signer + //! @param[in,out] signers vector to which new signers (with a unique master key fingerprint) are added + //! @param chain "main", "test", "regtest" or "signet" + //! @param[out] success Boolean + static bool Enumerate(const std::string& command, std::vector<ExternalSigner>& signers, std::string chain, bool ignore_errors = false); + + //! Display address on the device. Calls `<command> displayaddress --desc <descriptor>`. + //! @param[in] descriptor Descriptor specifying which address to display. + //! Must include a public key or xpub, as well as key origin. + UniValue DisplayAddress(const std::string& descriptor) const; + + //! Get receive and change Descriptor(s) from device for a given account. + //! Calls `<command> getdescriptors --account <account>` + //! @param[in] account which BIP32 account to use (e.g. `m/44'/0'/account'`) + //! @param[out] UniValue see doc/external-signer.md + UniValue GetDescriptors(int account); + + //! Sign PartiallySignedTransaction on the device. + //! Calls `<command> signtransaction` and passes the PSBT via stdin. + //! @param[in,out] psbt PartiallySignedTransaction to be signed + bool SignTransaction(PartiallySignedTransaction& psbt, std::string& error); + +#endif +}; + +#endif // BITCOIN_WALLET_EXTERNAL_SIGNER_H diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp new file mode 100644 index 0000000000..a2071e521a --- /dev/null +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -0,0 +1,81 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <wallet/external_signer.h> +#include <wallet/external_signer_scriptpubkeyman.h> + +#ifdef ENABLE_EXTERNAL_SIGNER + +bool ExternalSignerScriptPubKeyMan::SetupDescriptor(std::unique_ptr<Descriptor> desc) +{ + LOCK(cs_desc_man); + assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + assert(m_storage.IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)); + + int64_t creation_time = GetTime(); + + // Make the descriptor + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + m_wallet_descriptor = w_desc; + + // Store the descriptor + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } + + // TopUp + TopUp(); + + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { + const std::string command = gArgs.GetArg("-signer", ""); + if (command == "") throw std::runtime_error(std::string(__func__) + ": restart bitcoind with -signer=<cmd>"); + std::vector<ExternalSigner> signers; + ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); + // TODO: add fingerprint argument in case of multiple signers + return signers[0]; +} + +bool ExternalSignerScriptPubKeyMan::DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const +{ + // TODO: avoid the need to infer a descriptor from inside a descriptor wallet + auto provider = GetSolvingProvider(scriptPubKey); + auto descriptor = InferDescriptor(scriptPubKey, *provider); + + signer.DisplayAddress(descriptor->ToString()); + // TODO inspect result + return true; +} + +// If sign is true, transaction must previously have been filled +TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const +{ + if (!sign) { + return DescriptorScriptPubKeyMan::FillPSBT(psbt, sighash_type, false, bip32derivs, n_signed); + } + + // Already complete if every input is now signed + bool complete = true; + for (const auto& input : psbt.inputs) { + // TODO: for multisig wallets, we should only care if all _our_ inputs are signed + complete &= PSBTInputSigned(input); + } + if (complete) return TransactionError::OK; + + std::string strFailReason; + if(!GetExternalSigner().SignTransaction(psbt, strFailReason)) { + tfm::format(std::cerr, "Failed to sign: %s\n", strFailReason); + return TransactionError::EXTERNAL_SIGNER_FAILED; + } + FinalizePSBT(psbt); // This won't work in a multisig setup + return TransactionError::OK; +} + +#endif diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h new file mode 100644 index 0000000000..e60d7b8004 --- /dev/null +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -0,0 +1,34 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H +#define BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H + +#ifdef ENABLE_EXTERNAL_SIGNER +#include <wallet/scriptpubkeyman.h> + +class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan +{ + public: + ExternalSignerScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) + : DescriptorScriptPubKeyMan(storage, descriptor) + {} + ExternalSignerScriptPubKeyMan(WalletStorage& storage, bool internal) + : DescriptorScriptPubKeyMan(storage, internal) + {} + + /** Provide a descriptor at setup time + * Returns false if already setup or setup fails, true if setup is successful + */ + bool SetupDescriptor(std::unique_ptr<Descriptor>desc); + + static ExternalSigner GetExternalSigner(); + + bool DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const; + + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; +}; +#endif + +#endif // BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 085dde1026..f3e24384df 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -61,6 +61,9 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)", CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); +#ifdef ENABLE_EXTERNAL_SIGNER + argsman.AddArg("-signer=<cmd>", "External signing tool, see doc/external-signer.md", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); +#endif argsman.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-wallet=<path>", "Specify wallet path to load at startup. Can be used multiple times to load multiple wallets. Path is to a directory containing wallet data and log files. If the path is not absolute, it is interpreted relative to <walletdir>. This only loads existing wallets and does not create new ones. For backwards compatibility this also accepts names of existing top-level data files in <walletdir>.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -86,6 +89,11 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const bool WalletInit::ParameterInteraction() const { +#ifdef USE_BDB + if (!BerkeleyDatabaseSanityCheck()) { + return InitError(Untranslated("A version conflict was detected between the run-time BerkeleyDB library and the one used during compilation.")); + } +#endif if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { for (const std::string& wallet : gArgs.GetArgs("-wallet")) { LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet); diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index e4e8c50f4f..1fb789b128 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -23,6 +23,7 @@ #include <wallet/fees.h> #include <wallet/ismine.h> #include <wallet/load.h> +#include <wallet/rpcsigner.h> #include <wallet/rpcwallet.h> #include <wallet/wallet.h> @@ -518,6 +519,15 @@ public: }, command.argNames, command.unique_id); m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); } + +#ifdef ENABLE_EXTERNAL_SIGNER + for (const CRPCCommand& command : GetSignerRPCCommands()) { + m_rpc_commands.emplace_back(command.category, command.name, [this, &command](const JSONRPCRequest& request, UniValue& result, bool last_handler) { + return command.actor({request, m_context}, result, last_handler); + }, command.argNames, command.unique_id); + m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); + } +#endif } bool verify() override { return VerifyWallets(*m_context.chain); } bool load() override { return LoadWallets(*m_context.chain); } diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 5cdd7dff80..38ed7e7770 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -14,7 +14,27 @@ class CWallet; class CScript; -/** IsMine() return codes */ +/** + * IsMine() return codes, which depend on ScriptPubKeyMan implementation. + * Not every ScriptPubKeyMan covers all types, please refer to + * https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.21.0.md#ismine-semantics + * for better understanding. + * + * For LegacyScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_WATCH_ONLY: the scriptPubKey has been imported into the wallet; + * ISMINE_SPENDABLE: the scriptPubKey corresponds to an address owned by the wallet user (can spend with the private key); + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user; + * ISMINE_ALL: all ISMINE flags except for USED; + * ISMINE_ALL_USED: all ISMINE flags including USED; + * ISMINE_ENUM_ELEMENTS: the number of isminetype enum elements. + * + * For DescriptorScriptPubKeyMan and future ScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_SPENDABLE: the scriptPubKey matches a scriptPubKey in the wallet. + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user. + * + */ enum isminetype : unsigned int { ISMINE_NO = 0, diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 036fd4956f..30832f983b 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -62,7 +62,7 @@ bool VerifyWallets(interfaces::Chain& chain) std::set<fs::path> wallet_paths; for (const auto& wallet_file : gArgs.GetArgs("-wallet")) { - const fs::path path = fs::absolute(wallet_file, GetWalletDir()); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_file); if (!wallet_paths.insert(path).second) { chain.initWarning(strprintf(_("Ignoring duplicate -wallet %s."), wallet_file)); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6b46868d10..a2872f10ae 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -934,9 +934,9 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TxoutType::NONSTANDARD: case TxoutType::WITNESS_UNKNOWN: case TxoutType::WITNESS_V1_TAPROOT: - default: return "unrecognized script"; - } + } // no default case, so the compiler can warn about missing cases + CHECK_NONFATAL(false); } static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) @@ -1740,3 +1740,78 @@ RPCHelpMan importdescriptors() }, }; } + +RPCHelpMan listdescriptors() +{ + return RPCHelpMan{ + "listdescriptors", + "\nList descriptors imported into a descriptor-enabled wallet.", + {}, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array of descriptor objects", + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "desc", "Descriptor string representation"}, + {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, + {RPCResult::Type::BOOL, "active", "Activeness flag"}, + {RPCResult::Type::BOOL, "internal", true, "Whether this is internal or external descriptor; defined only for active descriptors"}, + {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { + {RPCResult::Type::NUM, "", "Range start inclusive"}, + {RPCResult::Type::NUM, "", "Range end inclusive"}, + }}, + {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("listdescriptors", "") + HelpExampleRpc("listdescriptors", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + + if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "listdescriptors is not available for non-descriptor wallets"); + } + + EnsureWalletIsUnlocked(wallet.get()); + + LOCK(wallet->cs_wallet); + + UniValue response(UniValue::VARR); + const auto active_spk_mans = wallet->GetActiveScriptPubKeyMans(); + for (const auto& spk_man : wallet->GetAllScriptPubKeyMans()) { + const auto desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); + if (!desc_spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "Unexpected ScriptPubKey manager type."); + } + UniValue spk(UniValue::VOBJ); + LOCK(desc_spk_man->cs_desc_man); + const auto& wallet_descriptor = desc_spk_man->GetWalletDescriptor(); + std::string descriptor; + if (!desc_spk_man->GetDescriptorString(descriptor, false)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't get normalized descriptor string."); + } + spk.pushKV("desc", descriptor); + spk.pushKV("timestamp", wallet_descriptor.creation_time); + const bool active = active_spk_mans.count(desc_spk_man) != 0; + spk.pushKV("active", active); + const auto& type = wallet_descriptor.descriptor->GetOutputType(); + if (active && type != nullopt) { + spk.pushKV("internal", wallet->GetScriptPubKeyMan(*type, true) == desc_spk_man); + } + if (wallet_descriptor.descriptor->IsRange()) { + UniValue range(UniValue::VARR); + range.push_back(wallet_descriptor.range_start); + range.push_back(wallet_descriptor.range_end - 1); + spk.pushKV("range", range); + spk.pushKV("next", wallet_descriptor.next_index); + } + response.push_back(spk); + } + + return response; +}, + }; +} diff --git a/src/wallet/rpcsigner.cpp b/src/wallet/rpcsigner.cpp new file mode 100644 index 0000000000..607b778c68 --- /dev/null +++ b/src/wallet/rpcsigner.cpp @@ -0,0 +1,111 @@ +// Copyright (c) 2018-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparamsbase.h> +#include <key_io.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <util/strencodings.h> +#include <wallet/rpcsigner.h> +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> + +#ifdef ENABLE_EXTERNAL_SIGNER + +static RPCHelpMan enumeratesigners() +{ + return RPCHelpMan{ + "enumeratesigners", + "Returns a list of external signers from -signer.", + {}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ARR, "signers", /* optional */ false, "", + { + {RPCResult::Type::STR_HEX, "masterkeyfingerprint", "Master key fingerprint"}, + {RPCResult::Type::STR, "name", "Device name"}, + }, + } + } + }, + RPCExamples{""}, + [](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + + const std::string command = gArgs.GetArg("-signer", ""); + if (command == "") throw JSONRPCError(RPC_WALLET_ERROR, "Error: restart bitcoind with -signer=<cmd>"); + std::string chain = gArgs.GetChainName(); + UniValue signers_res = UniValue::VARR; + try { + std::vector<ExternalSigner> signers; + ExternalSigner::Enumerate(command, signers, chain); + for (ExternalSigner signer : signers) { + UniValue signer_res = UniValue::VOBJ; + signer_res.pushKV("fingerprint", signer.m_fingerprint); + signer_res.pushKV("name", signer.m_name); + signers_res.push_back(signer_res); + } + } catch (const ExternalSignerException& e) { + throw JSONRPCError(RPC_WALLET_ERROR, e.what()); + } + UniValue result(UniValue::VOBJ); + result.pushKV("signers", signers_res); + return result; + } + }; +} + +static RPCHelpMan signerdisplayaddress() +{ + return RPCHelpMan{ + "signerdisplayaddress", + "Display address on an external signer for verification.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, /* default_val */ "", "bitcoin address to display"}, + }, + RPCResult{RPCResult::Type::NONE,"",""}, + RPCExamples{""}, + [](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + CWallet* const pwallet = wallet.get(); + + LOCK(pwallet->cs_wallet); + + CTxDestination dest = DecodeDestination(request.params[0].get_str()); + + // Make sure the destination is valid + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + if (!pwallet->DisplayAddress(dest)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Failed to display address"); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("address", request.params[0].get_str()); + return result; + } + }; +} + +Span<const CRPCCommand> GetSignerRPCCommands() +{ + +// clang-format off +static const CRPCCommand commands[] = +{ // category actor (function) + // --------------------- ------------------------ + { "signer", &enumeratesigners, }, + { "signer", &signerdisplayaddress, }, +}; +// clang-format on + return MakeSpan(commands); +} + + +#endif // ENABLE_EXTERNAL_SIGNER diff --git a/src/wallet/rpcsigner.h b/src/wallet/rpcsigner.h new file mode 100644 index 0000000000..f3ab83c428 --- /dev/null +++ b/src/wallet/rpcsigner.h @@ -0,0 +1,25 @@ +// Copyright (c) 2018-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_RPCSIGNER_H +#define BITCOIN_WALLET_RPCSIGNER_H + +#include <span.h> +#include <util/system.h> +#include <vector> + +#ifdef ENABLE_EXTERNAL_SIGNER + +class CRPCCommand; + +namespace interfaces { +class Chain; +class Handler; +} + +Span<const CRPCCommand> GetSignerRPCCommands(); + +#endif // ENABLE_EXTERNAL_SIGNER + +#endif //BITCOIN_WALLET_RPCSIGNER_H diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index ae4e8f2898..6d6cdedb2b 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -400,6 +400,12 @@ UniValue SendMoney(CWallet* const pwallet, const CCoinControl &coin_control, std { EnsureWalletIsUnlocked(pwallet); + // This function is only used by sendtoaddress and sendmany. + // This should always try to sign, if we don't have private keys, don't try to do anything here. + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); + } + // Shuffle recipient list std::shuffle(recipients.begin(), recipients.end(), FastRandomContext()); @@ -409,7 +415,7 @@ UniValue SendMoney(CWallet* const pwallet, const CCoinControl &coin_control, std bilingual_str error; CTransactionRef tx; FeeCalculation fee_calc_out; - bool fCreated = pwallet->CreateTransaction(recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, fee_calc_out, !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + const bool fCreated = pwallet->CreateTransaction(recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, fee_calc_out, true); if (!fCreated) { throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); } @@ -624,10 +630,10 @@ static RPCHelpMan signmessage() CTxDestination dest = DecodeDestination(strAddress); if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid address"); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } - const PKHash *pkhash = boost::get<PKHash>(&dest); + const PKHash* pkhash = std::get_if<PKHash>(&dest); if (!pkhash) { throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); } @@ -2615,7 +2621,18 @@ static RPCHelpMan loadwallet() if (!wallet) { // Map bad format to not found, since bad format is returned when the // wallet directory exists, but doesn't contain a data file. - RPCErrorCode code = status == DatabaseStatus::FAILED_NOT_FOUND || status == DatabaseStatus::FAILED_BAD_FORMAT ? RPC_WALLET_NOT_FOUND : RPC_WALLET_ERROR; + RPCErrorCode code = RPC_WALLET_ERROR; + switch (status) { + case DatabaseStatus::FAILED_NOT_FOUND: + case DatabaseStatus::FAILED_BAD_FORMAT: + code = RPC_WALLET_NOT_FOUND; + break; + case DatabaseStatus::FAILED_ALREADY_LOADED: + code = RPC_WALLET_ALREADY_LOADED; + break; + default: // RPC_WALLET_ERROR is returned for all other cases. + break; + } throw JSONRPCError(code, error.original); } @@ -2709,6 +2726,7 @@ static RPCHelpMan createwallet() {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "false", "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, {"descriptors", RPCArg::Type::BOOL, /* default */ "false", "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, {"load_on_startup", RPCArg::Type::BOOL, /* default */ "null", "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + {"external_signer", RPCArg::Type::BOOL, /* default */ "false", "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2753,6 +2771,13 @@ static RPCHelpMan createwallet() flags |= WALLET_FLAG_DESCRIPTORS; warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } + if (!request.params[7].isNull() && request.params[7].get_bool()) { +#ifdef ENABLE_EXTERNAL_SIGNER + flags |= WALLET_FLAG_EXTERNAL_SIGNER; +#else + throw JSONRPCError(RPC_WALLET_ERROR, "Configure with --enable-external-signer to use this"); +#endif + } #ifndef USE_BDB if (!(flags & WALLET_FLAG_DESCRIPTORS)) { @@ -3002,7 +3027,7 @@ static RPCHelpMan listunspent() std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); if (provider) { if (scriptPubKey.IsPayToScriptHash()) { - const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); + const CScriptID& hash = CScriptID(std::get<ScriptHash>(address)); CScript redeemScript; if (provider->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript)); @@ -3012,7 +3037,7 @@ static RPCHelpMan listunspent() bool extracted = ExtractDestination(redeemScript, witness_destination); CHECK_NONFATAL(extracted); // Also return the witness script - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); + const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(witness_destination); CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; @@ -3022,7 +3047,7 @@ static RPCHelpMan listunspent() } } } else if (scriptPubKey.IsPayToWitnessScriptHash()) { - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); + const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(address); CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; @@ -3449,10 +3474,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) CWallet* const pwallet = wallet.get(); if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { - if (!pwallet->chain().rpcEnableDeprecated("bumpfee")) { - throw JSONRPCError(RPC_METHOD_DEPRECATED, "Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22"); - } - want_psbt = true; + throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); @@ -3645,7 +3667,7 @@ static RPCHelpMan rescanblockchain() }; } -class DescribeWalletAddressVisitor : public boost::static_visitor<UniValue> +class DescribeWalletAddressVisitor { public: const SigningProvider * const provider; @@ -3664,7 +3686,7 @@ public: UniValue subobj(UniValue::VOBJ); UniValue detail = DescribeAddress(embedded); subobj.pushKVs(detail); - UniValue wallet_detail = boost::apply_visitor(*this, embedded); + UniValue wallet_detail = std::visit(*this, embedded); subobj.pushKVs(wallet_detail); subobj.pushKV("address", EncodeDestination(embedded)); subobj.pushKV("scriptPubKey", HexStr(subscript)); @@ -3747,7 +3769,7 @@ static UniValue DescribeWalletAddress(const CWallet* const pwallet, const CTxDes provider = pwallet->GetSolvingProvider(script); } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider.get()), dest)); + ret.pushKVs(std::visit(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -3779,6 +3801,7 @@ RPCHelpMan getaddressinfo() {RPCResult::Type::BOOL, "iswatchonly", "If the address is watchonly."}, {RPCResult::Type::BOOL, "solvable", "If we know how to spend coins sent to this address, ignoring the possible lack of private keys."}, {RPCResult::Type::STR, "desc", /* optional */ true, "A descriptor for spending coins sent to this address (only when solvable)."}, + {RPCResult::Type::STR, "parent_desc", /* optional */ true, "The descriptor used to derive this address if this is a descriptor wallet"}, {RPCResult::Type::BOOL, "isscript", "If the key is a script."}, {RPCResult::Type::BOOL, "ischange", "If the address was used for change output."}, {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address."}, @@ -3823,13 +3846,19 @@ RPCHelpMan getaddressinfo() LOCK(pwallet->cs_wallet); - UniValue ret(UniValue::VOBJ); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + // Make sure the destination is valid if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + // Set generic error message in case 'DecodeDestination' didn't set it + if (error_msg.empty()) error_msg = "Invalid address"; + + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error_msg); } + UniValue ret(UniValue::VOBJ); + std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -3848,6 +3877,14 @@ RPCHelpMan getaddressinfo() ret.pushKV("desc", InferDescriptor(scriptPubKey, *provider)->ToString()); } + DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(pwallet->GetScriptPubKeyMan(scriptPubKey)); + if (desc_spk_man) { + std::string desc_str; + if (desc_spk_man->GetDescriptorString(desc_str, false)) { + ret.pushKV("parent_desc", desc_str); + } + } + ret.pushKV("iswatchonly", bool(mine & ISMINE_WATCH_ONLY)); UniValue detail = DescribeWalletAddress(pwallet, dest); @@ -4158,8 +4195,10 @@ static RPCHelpMan send() // Make a blank psbt PartiallySignedTransaction psbtx(rawTx); - // Fill transaction with our data and sign - bool complete = true; + // First fill transaction with our data without signing, + // so external signers are not asked sign more than once. + bool complete; + pwallet->FillPSBT(psbtx, complete, SIGHASH_ALL, false, true); const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_ALL, true, false); if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); @@ -4537,73 +4576,75 @@ RPCHelpMan importprunedfunds(); RPCHelpMan removeprunedfunds(); RPCHelpMan importmulti(); RPCHelpMan importdescriptors(); +RPCHelpMan listdescriptors(); Span<const CRPCCommand> GetWalletRPCCommands() { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, - { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, - { "wallet", "abortrescan", &abortrescan, {} }, - { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, - { "wallet", "backupwallet", &backupwallet, {"destination"} }, - { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "psbtbumpfee", &psbtbumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors", "load_on_startup"} }, - { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, - { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, - { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, - { "wallet", "getaddressesbylabel", &getaddressesbylabel, {"label"} }, - { "wallet", "getaddressinfo", &getaddressinfo, {"address"} }, - { "wallet", "getbalance", &getbalance, {"dummy","minconf","include_watchonly","avoid_reuse"} }, - { "wallet", "getnewaddress", &getnewaddress, {"label","address_type"} }, - { "wallet", "getrawchangeaddress", &getrawchangeaddress, {"address_type"} }, - { "wallet", "getreceivedbyaddress", &getreceivedbyaddress, {"address","minconf"} }, - { "wallet", "getreceivedbylabel", &getreceivedbylabel, {"label","minconf"} }, - { "wallet", "gettransaction", &gettransaction, {"txid","include_watchonly","verbose"} }, - { "wallet", "getunconfirmedbalance", &getunconfirmedbalance, {} }, - { "wallet", "getbalances", &getbalances, {} }, - { "wallet", "getwalletinfo", &getwalletinfo, {} }, - { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, - { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, - { "wallet", "importmulti", &importmulti, {"requests","options"} }, - { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, - { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, - { "wallet", "importpubkey", &importpubkey, {"pubkey","label","rescan"} }, - { "wallet", "importwallet", &importwallet, {"filename"} }, - { "wallet", "keypoolrefill", &keypoolrefill, {"newsize"} }, - { "wallet", "listaddressgroupings", &listaddressgroupings, {} }, - { "wallet", "listlabels", &listlabels, {"purpose"} }, - { "wallet", "listlockunspent", &listlockunspent, {} }, - { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, {"minconf","include_empty","include_watchonly","address_filter"} }, - { "wallet", "listreceivedbylabel", &listreceivedbylabel, {"minconf","include_empty","include_watchonly"} }, - { "wallet", "listsinceblock", &listsinceblock, {"blockhash","target_confirmations","include_watchonly","include_removed"} }, - { "wallet", "listtransactions", &listtransactions, {"label|dummy","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, - { "wallet", "listwalletdir", &listwalletdir, {} }, - { "wallet", "listwallets", &listwallets, {} }, - { "wallet", "loadwallet", &loadwallet, {"filename", "load_on_startup"} }, - { "wallet", "lockunspent", &lockunspent, {"unlock","transactions"} }, - { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, - { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, - { "wallet", "send", &send, {"outputs","conf_target","estimate_mode","fee_rate","options"} }, - { "wallet", "sendmany", &sendmany, {"dummy","amounts","minconf","comment","subtractfeefrom","replaceable","conf_target","estimate_mode","fee_rate","verbose"} }, - { "wallet", "sendtoaddress", &sendtoaddress, {"address","amount","comment","comment_to","subtractfeefromamount","replaceable","conf_target","estimate_mode","avoid_reuse","fee_rate","verbose"} }, - { "wallet", "sethdseed", &sethdseed, {"newkeypool","seed"} }, - { "wallet", "setlabel", &setlabel, {"address","label"} }, - { "wallet", "settxfee", &settxfee, {"amount"} }, - { "wallet", "setwalletflag", &setwalletflag, {"flag","value"} }, - { "wallet", "signmessage", &signmessage, {"address","message"} }, - { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, - { "wallet", "unloadwallet", &unloadwallet, {"wallet_name", "load_on_startup"} }, - { "wallet", "upgradewallet", &upgradewallet, {"version"} }, - { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, - { "wallet", "walletlock", &walletlock, {} }, - { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, - { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, - { "wallet", "walletprocesspsbt", &walletprocesspsbt, {"psbt","sign","sighashtype","bip32derivs"} }, +{ // category actor (function) + // ------------------ ------------------------ + { "rawtransactions", &fundrawtransaction, }, + { "wallet", &abandontransaction, }, + { "wallet", &abortrescan, }, + { "wallet", &addmultisigaddress, }, + { "wallet", &backupwallet, }, + { "wallet", &bumpfee, }, + { "wallet", &psbtbumpfee, }, + { "wallet", &createwallet, }, + { "wallet", &dumpprivkey, }, + { "wallet", &dumpwallet, }, + { "wallet", &encryptwallet, }, + { "wallet", &getaddressesbylabel, }, + { "wallet", &getaddressinfo, }, + { "wallet", &getbalance, }, + { "wallet", &getnewaddress, }, + { "wallet", &getrawchangeaddress, }, + { "wallet", &getreceivedbyaddress, }, + { "wallet", &getreceivedbylabel, }, + { "wallet", &gettransaction, }, + { "wallet", &getunconfirmedbalance, }, + { "wallet", &getbalances, }, + { "wallet", &getwalletinfo, }, + { "wallet", &importaddress, }, + { "wallet", &importdescriptors, }, + { "wallet", &importmulti, }, + { "wallet", &importprivkey, }, + { "wallet", &importprunedfunds, }, + { "wallet", &importpubkey, }, + { "wallet", &importwallet, }, + { "wallet", &keypoolrefill, }, + { "wallet", &listaddressgroupings, }, + { "wallet", &listdescriptors, }, + { "wallet", &listlabels, }, + { "wallet", &listlockunspent, }, + { "wallet", &listreceivedbyaddress, }, + { "wallet", &listreceivedbylabel, }, + { "wallet", &listsinceblock, }, + { "wallet", &listtransactions, }, + { "wallet", &listunspent, }, + { "wallet", &listwalletdir, }, + { "wallet", &listwallets, }, + { "wallet", &loadwallet, }, + { "wallet", &lockunspent, }, + { "wallet", &removeprunedfunds, }, + { "wallet", &rescanblockchain, }, + { "wallet", &send, }, + { "wallet", &sendmany, }, + { "wallet", &sendtoaddress, }, + { "wallet", &sethdseed, }, + { "wallet", &setlabel, }, + { "wallet", &settxfee, }, + { "wallet", &setwalletflag, }, + { "wallet", &signmessage, }, + { "wallet", &signrawtransactionwithwallet, }, + { "wallet", &unloadwallet, }, + { "wallet", &upgradewallet, }, + { "wallet", &walletcreatefundedpsbt, }, + { "wallet", &walletlock, }, + { "wallet", &walletpassphrase, }, + { "wallet", &walletpassphrasechange, }, + { "wallet", &walletprocesspsbt, }, }; // clang-format on return MakeSpan(commands); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 15972fe7bb..efb408c163 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -13,6 +13,7 @@ #include <util/system.h> #include <util/time.h> #include <util/translation.h> +#include <wallet/external_signer.h> #include <wallet/scriptpubkeyman.h> //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. @@ -94,8 +95,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s TxoutType whichType = Solver(scriptPubKey, vSolutions); CKeyID keyID; - switch (whichType) - { + switch (whichType) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -194,7 +194,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s } break; } - } + } // no default case, so the compiler can warn about missing cases if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { ret = std::max(ret, IsMineResult::WATCH_ONLY); @@ -2265,3 +2265,16 @@ const std::vector<CScript> DescriptorScriptPubKeyMan::GetScriptPubKeys() const } return script_pub_keys; } + +bool DescriptorScriptPubKeyMan::GetDescriptorString(std::string& out, bool priv) const +{ + LOCK(cs_desc_man); + if (m_storage.IsLocked()) { + return false; + } + + FlatSigningProvider provider; + provider.keys = GetKeys(); + + return m_wallet_descriptor.descriptor->ToNormalizedString(provider, out, priv); +} diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index cec46a0fbb..b8e34fbac3 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -304,7 +304,7 @@ private: /* the HD chain data model (external chain counters) */ CHDChain m_hd_chain; - std::unordered_map<CKeyID, CHDChain, KeyIDHasher> m_inactive_hd_chains; + std::unordered_map<CKeyID, CHDChain, SaltedSipHasher> m_inactive_hd_chains; /* HD derive new child key (on internal or external chain) */ void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); @@ -517,8 +517,6 @@ public: class DescriptorScriptPubKeyMan : public ScriptPubKeyMan { private: - WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); - using ScriptPubKeyMap = std::map<CScript, int32_t>; // Map of scripts to descriptor range index using PubKeyMap = std::map<CPubKey, int32_t>; // Map of pubkeys involved in scripts to descriptor range index using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; @@ -547,6 +545,9 @@ private: // Fetch the SigningProvider for a given index and optionally include private keys. Called by the above functions. std::unique_ptr<FlatSigningProvider> GetSigningProvider(int32_t index, bool include_private = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); +protected: + WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + public: DescriptorScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) : ScriptPubKeyMan(storage), @@ -581,6 +582,11 @@ public: //! Setup descriptors based on the given CExtkey bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type); + /** Provide a descriptor at setup time + * Returns false if already setup or setup fails, true if setup is successful + */ + bool SetupDescriptor(std::unique_ptr<Descriptor>desc); + bool HavePrivateKeys() const override; int64_t GetOldestKeyPoolTime() const override; @@ -616,6 +622,8 @@ public: const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); const std::vector<CScript> GetScriptPubKeys() const; + + bool GetDescriptorString(std::string& out, bool priv) const; }; #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 202804c9ff..ffac78d752 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -35,7 +35,7 @@ static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); CoinEligibilityFilter filter_confirmed(1, 1, 0); CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0); +CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0, false); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -115,7 +115,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<CInputCoin>& coins { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin, 0, true, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin, 0, true, 0, 0, false); + } return static_groups; } @@ -123,7 +126,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0, false); + } return static_groups; } @@ -263,22 +269,22 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } // Make sure that effective value is working in SelectCoinsMinConf when BnB is used - CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0); + CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0, false); CoinSet setCoinsRet; CAmount nValueRet; bool bnb_used; empty_wallet(); add_coin(1); vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); // Test fees subtracted from output: empty_wallet(); add_coin(1 * CENT); vCoins.at(0).nInputBytes = 40; - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); // Make sure that can use BnB when there are preset inputs @@ -317,24 +323,24 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) empty_wallet(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); add_coin(1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can find a new 1 cent - BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); add_coin(2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can make 3 cents of new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); add_coin(5*CENT); // add a mature 5 cent coin, @@ -344,33 +350,33 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -384,30 +390,30 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -416,11 +422,11 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin( 2*COIN); add_coin( 3*COIN); add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -435,14 +441,14 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided add_coin(1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: @@ -450,7 +456,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) @@ -459,7 +465,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(50000 * COIN); - BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -472,7 +478,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 7 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -482,7 +488,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 8 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 @@ -493,12 +499,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); } @@ -512,7 +518,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); if (amt - 2000 < MIN_CHANGE) { // needs more than one input: @@ -538,8 +544,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -547,8 +553,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -570,8 +576,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -598,7 +604,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(1000 * COIN); add_coin(3 * COIN); - BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -633,13 +639,13 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CAmount target = rand.randrange(balance - 1000) + 1000; // Perform selection - CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0); - CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0); + CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0, false); + CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0, false); CoinSet out_set; CAmount out_value = 0; bool bnb_used = false; - BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_bnb, bnb_used) || - testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_knapsack, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_bnb, bnb_used) || + testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_knapsack, bnb_used)); BOOST_CHECK_GE(out_value, target); } } diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 27179839b7..b2eb8e4bca 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -30,8 +30,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - BOOST_CHECK(filename == test_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, test_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_directory) @@ -41,8 +41,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_directory) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename); - BOOST_CHECK(filename == expected_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, expected_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index aa7f1c83d8..f035a70a20 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <fs.h> +#include <univalue.h> #include <util/check.h> #include <util/system.h> @@ -37,6 +38,9 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam InitWalletDirTestingSetup::~InitWalletDirTestingSetup() { + gArgs.LockSettings([&](util::Settings& settings) { + settings.forced_settings.erase("walletdir"); + }); fs::current_path(m_cwd); } diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index e70b56c529..45e1b8c4b8 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -19,7 +19,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a6db261914..5480f3ab22 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -550,7 +550,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 1U); // Check initial balance from one mature coinbase transaction. @@ -566,7 +566,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 2U); // Lock both coins. Confirm number of available coins drops to 0. @@ -595,7 +595,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 2U); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0bcbff3b2f..08e480225d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -14,10 +14,12 @@ #include <key.h> #include <key_io.h> #include <optional.h> +#include <outputtype.h> #include <policy/fees.h> #include <policy/policy.h> #include <primitives/block.h> #include <primitives/transaction.h> +#include <psbt.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> @@ -32,6 +34,7 @@ #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> +#include <wallet/external_signer_scriptpubkeyman.h> #include <univalue.h> @@ -259,6 +262,20 @@ std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::strin wallet_creation_flags |= WALLET_FLAG_BLANK_WALLET; } + // Private keys must be disabled for an external signer wallet + if ((wallet_creation_flags & WALLET_FLAG_EXTERNAL_SIGNER) && !(wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + error = Untranslated("Private keys must be disabled when using an external signer"); + status = DatabaseStatus::FAILED_CREATE; + return nullptr; + } + + // Descriptor support must be enabled for an external signer wallet + if ((wallet_creation_flags & WALLET_FLAG_EXTERNAL_SIGNER) && !(wallet_creation_flags & WALLET_FLAG_DESCRIPTORS)) { + error = Untranslated("Descriptor support must be enabled when using an external signer"); + status = DatabaseStatus::FAILED_CREATE; + return nullptr; + } + // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); if (!database) { @@ -2357,13 +2374,12 @@ const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int out return ptx->vout[n]; } -bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, +bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const { setCoinsRet.clear(); nValueRet = 0; - std::vector<OutputGroup> utxo_pool; if (coin_selection_params.use_bnb) { // Get long term estimate FeeCalculation feeCalc; @@ -2371,35 +2387,27 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil temp.m_confirm_target = 1008; CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); - // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + // Get the feerate for effective value. + // When subtracting the fee from the outputs, we want the effective feerate to be 0 + CFeeRate effective_feerate{0}; + if (!coin_selection_params.m_subtract_fee_outputs) { + effective_feerate = coin_selection_params.effective_fee; + } - // Filter by the min conf specs and add to utxo_pool and calculate effective value - for (OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, effective_feerate, long_term_feerate, eligibility_filter, true /* positive_only */); - if (coin_selection_params.m_subtract_fee_outputs) { - // Set the effective feerate to 0 as we don't want to use the effective value since the fees will be deducted from the output - group.SetFees(CFeeRate(0) /* effective_feerate */, long_term_feerate); - } else { - group.SetFees(coin_selection_params.effective_fee, long_term_feerate); - } + // Calculate cost of change + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); - OutputGroup pos_group = group.GetPositiveOnlyGroup(); - if (pos_group.effective_value > 0) utxo_pool.push_back(pos_group); - } // Calculate the fees for things that aren't inputs CAmount not_input_fees = coin_selection_params.effective_fee.GetFee(coin_selection_params.tx_noinputs_size); bnb_used = true; - return SelectCoinsBnB(utxo_pool, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); + return SelectCoinsBnB(groups, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); } else { - // Filter by the min conf specs and add to utxo_pool - for (const OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; - utxo_pool.push_back(group); - } + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, CFeeRate(0), CFeeRate(0), eligibility_filter, false /* positive_only */); + bnb_used = false; - return KnapsackSolver(nTargetValue, utxo_pool, setCoinsRet, nValueRet); + return KnapsackSolver(nTargetValue, groups, setCoinsRet, nValueRet); } } @@ -2482,16 +2490,14 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); - bool res = value_to_select <= 0 || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset util::insert(setCoinsRet, setPresetCoins); @@ -2782,6 +2788,7 @@ bool CWallet::CreateTransactionInternal( std::vector<COutput> vAvailableCoins; AvailableCoins(vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy + coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; // Create change script that will be used if we need change // TODO: pass in scriptChange instead of reservedest so @@ -2789,7 +2796,7 @@ bool CWallet::CreateTransactionInternal( CScript scriptChange; // coin control: send change to custom address - if (!boost::get<CNoDestination>(&coin_control.destChange)) { + if (!std::get_if<CNoDestination>(&coin_control.destChange)) { scriptChange = GetScriptForDestination(coin_control.destChange); } else { // no coin control: send change to newly generated address // Note: We use a new key here to keep it from being obvious which side is the change. @@ -3568,6 +3575,38 @@ void ReserveDestination::ReturnDestination() address = CNoDestination(); } +#ifdef ENABLE_EXTERNAL_SIGNER +ExternalSigner CWallet::GetExternalSigner() +{ + const std::string command = gArgs.GetArg("-signer", ""); + if (command == "") throw std::runtime_error(std::string(__func__) + ": restart bitcoind with -signer=<cmd>"); + std::vector<ExternalSigner> signers; + ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); + // TODO: add fingerprint argument in case of multiple signers + return signers[0]; +} +#endif + +bool CWallet::DisplayAddress(const CTxDestination& dest) +{ +#ifdef ENABLE_EXTERNAL_SIGNER + CScript scriptPubKey = GetScriptForDestination(dest); + const auto spk_man = GetScriptPubKeyMan(scriptPubKey); + if (spk_man == nullptr) { + return false; + } + auto signer_spk_man = dynamic_cast<ExternalSignerScriptPubKeyMan*>(spk_man); + if (signer_spk_man == nullptr) { + return false; + } + ExternalSigner signer = GetExternalSigner(); // TODO: move signer in spk_man + return signer_spk_man->DisplayAddress(scriptPubKey, signer); +#else + return false; +#endif +} + void CWallet::LockCoin(const COutPoint& output) { AssertLockHeld(cs_wallet); @@ -3724,7 +3763,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key, const std::string &value) { - if (boost::get<CNoDestination>(&dest)) + if (std::get_if<CNoDestination>(&dest)) return false; m_address_book[dest].destdata.insert(std::make_pair(key, value)); @@ -3780,7 +3819,7 @@ std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, cons // 2. Path to an existing directory. // 3. Path to a symlink to a directory. // 4. For backwards compatibility, the name of a data file in -walletdir. - const fs::path& wallet_path = fs::absolute(name, GetWalletDir()); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), name); fs::file_type path_type = fs::symlink_status(wallet_path).type(); if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || @@ -3846,7 +3885,7 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain& chain, const std::st walletInstance->SetupLegacyScriptPubKeyMan(); } - if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { + if ((wallet_creation_flags & WALLET_FLAG_EXTERNAL_SIGNER) || !(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { LOCK(walletInstance->cs_wallet); if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { walletInstance->SetupDescriptorScriptPubKeyMans(); @@ -4192,51 +4231,90 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { - std::vector<OutputGroup> groups; - std::map<CTxDestination, OutputGroup> gmap; - std::set<CTxDestination> full_groups; +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const +{ + std::vector<OutputGroup> groups_out; - for (const auto& output : outputs) { - if (output.fSpendable) { - CTxDestination dst; - CInputCoin input_coin = output.GetInputCoin(); + if (separate_coins) { + // Single coin means no grouping. Each COutput gets its own OutputGroup. + for (const COutput& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - auto it = gmap.find(dst); - if (it != gmap.end()) { - // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES - // number of entries, to protect against inadvertently creating - // a too-large transaction when using -avoidpartialspends to - // prevent breaking consensus or surprising users with a very - // high amount of fees. - if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(it->second); - it->second = OutputGroup{}; - full_groups.insert(dst); - } - it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } else { - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } - } else { - groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } + CInputCoin input_coin = output.GetInputCoin(); + + // Make an OutputGroup containing just this output + OutputGroup group{effective_feerate, long_term_feerate}; + group.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); + } + return groups_out; + } + + // We want to combine COutputs that have the same scriptPubKey into single OutputGroups + // except when there are more than OUTPUT_GROUP_MAX_ENTRIES COutputs grouped in an OutputGroup. + // To do this, we maintain a map where the key is the scriptPubKey and the value is a vector of OutputGroups. + // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput's CInputCoin is added + // to the last OutputGroup in the vector for the scriptPubKey. When the last OutputGroup has + // OUTPUT_GROUP_MAX_ENTRIES CInputCoins, a new OutputGroup is added to the end of the vector. + std::map<CScript, std::vector<OutputGroup>> spk_to_groups_map; + for (const auto& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; + + size_t ancestors, descendants; + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + CInputCoin input_coin = output.GetInputCoin(); + CScript spk = input_coin.txout.scriptPubKey; + + std::vector<OutputGroup>& groups = spk_to_groups_map[spk]; + + if (groups.size() == 0) { + // No OutputGroups for this scriptPubKey yet, add one + groups.emplace_back(effective_feerate, long_term_feerate); + } + + // Get the last OutputGroup in the vector so that we can add the CInputCoin to it + // A pointer is used here so that group can be reassigned later if it is full. + OutputGroup* group = &groups.back(); + + // Check if this OutputGroup is full. We limit to OUTPUT_GROUP_MAX_ENTRIES when using -avoidpartialspends + // to avoid surprising users with very high fees. + if (group->m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + // The last output group is full, add a new group to the vector and use that group for the insertion + groups.emplace_back(effective_feerate, long_term_feerate); + group = &groups.back(); } + + // Add the input_coin to group + group->Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); } - if (!single_coin) { - for (auto& it : gmap) { - auto& group = it.second; - if (full_groups.count(it.first) > 0) { - // Make this unattractive as we want coin selection to avoid it if possible - group.m_ancestors = max_ancestors - 1; + + // Now we go through the entire map and pull out the OutputGroups + for (const auto& spk_and_groups_pair: spk_to_groups_map) { + const std::vector<OutputGroup>& groups_per_spk= spk_and_groups_pair.second; + + // Go through the vector backwards. This allows for the first item we deal with being the partial group. + for (auto group_it = groups_per_spk.rbegin(); group_it != groups_per_spk.rend(); group_it++) { + const OutputGroup& group = *group_it; + + // Don't include partial groups if there are full groups too and we don't want partial groups + if (group_it == groups_per_spk.rbegin() && groups_per_spk.size() > 1 && !filter.m_include_partial_groups) { + continue; } - groups.push_back(group); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); } } - return groups; + + return groups_out; } bool CWallet::IsCrypted() const @@ -4414,40 +4492,82 @@ void CWallet::ConnectScriptPubKeyManNotifiers() void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) { - auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); - m_spk_managers[id] = std::move(spk_manager); + if (IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { +#ifdef ENABLE_EXTERNAL_SIGNER + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, desc)); + m_spk_managers[id] = std::move(spk_manager); +#else + throw std::runtime_error(std::string(__func__) + ": Configure with --enable-external-signer to use external signer wallets"); +#endif + } else { + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + m_spk_managers[id] = std::move(spk_manager); + } } void CWallet::SetupDescriptorScriptPubKeyMans() { AssertLockHeld(cs_wallet); - // Make a seed - CKey seed_key; - seed_key.MakeNewKey(true); - CPubKey seed = seed_key.GetPubKey(); - assert(seed_key.VerifyPubKey(seed)); + if (!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + // Make a seed + CKey seed_key; + seed_key.MakeNewKey(true); + CPubKey seed = seed_key.GetPubKey(); + assert(seed_key.VerifyPubKey(seed)); - // Get the extended key - CExtKey master_key; - master_key.SetSeed(seed_key.begin(), seed_key.size()); + // Get the extended key + CExtKey master_key; + master_key.SetSeed(seed_key.begin(), seed_key.size()); - for (bool internal : {false, true}) { - for (OutputType t : OUTPUT_TYPES) { - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, internal)); - if (IsCrypted()) { - if (IsLocked()) { - throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, internal)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } } - if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { - throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + spk_manager->SetupDescriptorGeneration(master_key, t); + uint256 id = spk_manager->GetID(); + m_spk_managers[id] = std::move(spk_manager); + AddActiveScriptPubKeyMan(id, t, internal); + } + } + } else { +#ifdef ENABLE_EXTERNAL_SIGNER + ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); + + // TODO: add account parameter + int account = 0; + UniValue signer_res = signer.GetDescriptors(account); + + if (!signer_res.isObject()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); + for (bool internal : {false, true}) { + const UniValue& descriptor_vals = find_value(signer_res, internal ? "internal" : "receive"); + if (!descriptor_vals.isArray()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); + for (const UniValue& desc_val : descriptor_vals.get_array().getValues()) { + std::string desc_str = desc_val.getValStr(); + FlatSigningProvider keys; + std::string dummy_error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, dummy_error, false); + if (!desc->GetOutputType()) { + continue; } + OutputType t = *desc->GetOutputType(); + auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, internal)); + spk_manager->SetupDescriptor(std::move(desc)); + uint256 id = spk_manager->GetID(); + m_spk_managers[id] = std::move(spk_manager); + AddActiveScriptPubKeyMan(id, t, internal); } - spk_manager->SetupDescriptorGeneration(master_key, t); - uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); - AddActiveScriptPubKeyMan(id, t, internal); } +#else + throw std::runtime_error(std::string(__func__) + ": Wallets with external signers require Boost::Process library."); +#endif } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e3eae1dd95..eb797938cd 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -22,6 +22,7 @@ #include <wallet/coinselection.h> #include <wallet/crypter.h> #include <wallet/scriptpubkeyman.h> +#include <wallet/external_signer.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -95,7 +96,6 @@ constexpr CAmount DEFAULT_TRANSACTION_MAXFEE{COIN / 10}; constexpr CAmount HIGH_TX_FEE_PER_KB{COIN / 100}; //! -maxtxfee will warn if called with a higher fee than this amount (in satoshis) constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; - //! Pre-calculated constants for input size estimation in *virtual size* static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; @@ -115,7 +115,8 @@ static constexpr uint64_t KNOWN_WALLET_FLAGS = | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA | WALLET_FLAG_DISABLE_PRIVATE_KEYS - | WALLET_FLAG_DESCRIPTORS; + | WALLET_FLAG_DESCRIPTORS + | WALLET_FLAG_EXTERNAL_SIGNER; static constexpr uint64_t MUTABLE_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE; @@ -126,6 +127,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ {"key_origin_metadata", WALLET_FLAG_KEY_ORIGIN_METADATA}, {"disable_private_keys", WALLET_FLAG_DISABLE_PRIVATE_KEYS}, {"descriptor_wallet", WALLET_FLAG_DESCRIPTORS}, + {"external_signer", WALLET_FLAG_EXTERNAL_SIGNER} }; extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; @@ -610,8 +612,16 @@ struct CoinSelectionParams size_t tx_noinputs_size = 0; //! Indicate that we are subtracting the fee from outputs bool m_subtract_fee_outputs = false; - - CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size) : use_bnb(use_bnb), change_output_size(change_output_size), change_spend_size(change_spend_size), effective_fee(effective_fee), tx_noinputs_size(tx_noinputs_size) {} + bool m_avoid_partial_spends = false; + + CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size, bool avoid_partial) : + use_bnb(use_bnb), + change_output_size(change_output_size), + change_spend_size(change_spend_size), + effective_fee(effective_fee), + tx_noinputs_size(tx_noinputs_size), + m_avoid_partial_spends(avoid_partial) + {} CoinSelectionParams() {} }; @@ -818,7 +828,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, + bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -827,7 +837,13 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const; + +#ifdef ENABLE_EXTERNAL_SIGNER + ExternalSigner GetExternalSigner() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); +#endif + /** Display address on an external signer. Returns false if external signer support is not compiled */ + bool DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 4e6270220e..69854cae05 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -551,13 +551,6 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CHDChain chain; ssValue >> chain; pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadHDChain(chain); - } else if (strType == DBKeys::FLAGS) { - uint64_t flags; - ssValue >> flags; - if (!pwallet->LoadWalletFlags(flags)) { - strErr = "Error reading wallet database: Unknown non-tolerable wallet flags found"; - return false; - } } else if (strType == DBKeys::OLD_KEY) { strErr = "Found unsupported 'wkey' record, try loading with version 0.18"; return false; @@ -662,7 +655,8 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, wss.fIsEncrypted = true; } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && - strType != DBKeys::VERSION && strType != DBKeys::SETTINGS) { + strType != DBKeys::VERSION && strType != DBKeys::SETTINGS && + strType != DBKeys::FLAGS) { wss.m_unknown_records++; } } catch (const std::exception& e) { @@ -707,6 +701,16 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) pwallet->LoadMinVersion(nMinVersion); } + // Load wallet flags, so they are known when processing other records. + // The FLAGS key is absent during wallet creation. + uint64_t flags; + if (m_batch->Read(DBKeys::FLAGS, flags)) { + if (!pwallet->LoadWalletFlags(flags)) { + pwallet->WalletLogPrintf("Error reading wallet database: Unknown non-tolerable wallet flags found\n"); + return DBErrors::CORRUPT; + } + } + // Get cursor if (!m_batch->StartCursor()) { diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index a1bb7343f4..b2cb0bf479 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -103,10 +103,8 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& name) +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) { - fs::path path = fs::absolute(name, GetWalletDir()); - if (args.IsArgSet("-format") && command != "createfromdump") { tfm::format(std::cerr, "The -format option can only be used with the \"createfromdump\" command.\n"); return false; @@ -119,6 +117,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (command == "create" && !args.IsArgSet("-wallet")) { + tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + return false; + } + const std::string name = args.GetArg("-wallet", ""); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); if (command == "create") { DatabaseOptions options; diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index f544a6f727..f4516bb5bc 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -10,7 +10,7 @@ namespace WalletTool { void WalletShowInfo(CWallet* wallet_instance); -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& file); +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command); } // namespace WalletTool diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 67b2ee2b98..0713f768c1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -60,6 +60,9 @@ enum WalletFlags : uint64_t { //! Indicate that this wallet supports DescriptorScriptPubKeyMan WALLET_FLAG_DESCRIPTORS = (1ULL << 34), + + //! Indicates that the wallet needs an external signer + WALLET_FLAG_EXTERNAL_SIGNER = (1ULL << 35), }; //! Get the path of the wallet directory. diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index c0207f9dd6..168ba841c8 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -17,6 +17,7 @@ #include <cstdarg> #include <cstddef> #include <map> +#include <optional> #include <string> #include <utility> @@ -227,50 +228,43 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } +// Helper function to send a 'sequence' topic message with the following structure: +// <32-byte hash> | <1-byte label> | <8-byte LE sequence> (optional) +static bool SendSequenceMsg(CZMQAbstractPublishNotifier& notifier, uint256 hash, char label, std::optional<uint64_t> sequence = {}) +{ + unsigned char data[sizeof(hash) + sizeof(label) + sizeof(uint64_t)]; + for (unsigned int i = 0; i < sizeof(hash); ++i) { + data[sizeof(hash) - 1 - i] = hash.begin()[i]; + } + data[sizeof(hash)] = label; + if (sequence) WriteLE64(data + sizeof(hash) + sizeof(label), *sequence); + return notifier.SendZmqMessage(MSG_SEQUENCE, data, sequence ? sizeof(data) : sizeof(hash) + sizeof(label)); +} -// TODO: Dedup this code to take label char, log string bool CZMQPublishSequenceNotifier::NotifyBlockConnect(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); LogPrint(BCLog::ZMQ, "zmq: Publish sequence block connect %s to %s\n", hash.GetHex(), this->address); - char data[sizeof(uint256)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(data) - 1] = 'C'; // Block (C)onnect - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Block (C)onnect */ 'C'); } bool CZMQPublishSequenceNotifier::NotifyBlockDisconnect(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); LogPrint(BCLog::ZMQ, "zmq: Publish sequence block disconnect %s to %s\n", hash.GetHex(), this->address); - char data[sizeof(uint256)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(data) - 1] = 'D'; // Block (D)isconnect - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Block (D)isconnect */ 'D'); } bool CZMQPublishSequenceNotifier::NotifyTransactionAcceptance(const CTransaction &transaction, uint64_t mempool_sequence) { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "zmq: Publish hashtx mempool acceptance %s to %s\n", hash.GetHex(), this->address); - unsigned char data[sizeof(uint256)+sizeof(mempool_sequence)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(uint256)] = 'A'; // Mempool (A)cceptance - WriteLE64(data+sizeof(uint256)+1, mempool_sequence); - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Mempool (A)cceptance */ 'A', mempool_sequence); } bool CZMQPublishSequenceNotifier::NotifyTransactionRemoval(const CTransaction &transaction, uint64_t mempool_sequence) { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "zmq: Publish hashtx mempool removal %s to %s\n", hash.GetHex(), this->address); - unsigned char data[sizeof(uint256)+sizeof(mempool_sequence)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(uint256)] = 'R'; // Mempool (R)emoval - WriteLE64(data+sizeof(uint256)+1, mempool_sequence); - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Mempool (R)emoval */ 'R', mempool_sequence); } diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 1dd751b493..81859f924f 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -52,9 +52,9 @@ static RPCHelpMan getzmqnotifications() } const CRPCCommand commands[] = -{ // category name actor (function) argNames - // ----------------- ------------------------ ----------------------- ---------- - { "zmq", "getzmqnotifications", &getzmqnotifications, {} }, +{ // category actor (function) + // ----------------- ----------------------- + { "zmq", &getzmqnotifications, }, }; } // anonymous namespace diff --git a/test/config.ini.in b/test/config.ini.in index 77c9a720c3..e3872181cd 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -23,3 +23,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true +@ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true diff --git a/test/functional/README.md b/test/functional/README.md index 2d04413eb2..d830ba0334 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -63,10 +63,13 @@ don't have test cases for. - Avoid stop-starting the nodes multiple times during the test if possible. A stop-start takes several seconds, so doing it several times blows up the runtime of the test. -- Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether - or not to use the cached data directories. The cached data directories - contain a 200-block pre-mined blockchain and wallets for four nodes. Each node - has 25 mature blocks (25x50=1250 BTC) in its wallet. +- Set the `self.setup_clean_chain` variable in `set_test_params()` to `True` to + initialize an empty blockchain and start from the Genesis block, rather than + load a premined blockchain from cache with the default value of `False`. The + cached data directories contain a 200-block pre-mined blockchain with the + spendable mining rewards being split between four nodes. Each node has 25 + mature block subsidies (25x50=1250 BTC) in its wallet. Using them is much more + efficient than mining blocks in your test. - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 6e72db1d96..fab921ef19 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -57,7 +57,7 @@ class BadTxTemplate: __metaclass__ = abc.ABCMeta # The expected error code given by bitcoind upon submission of the tx. - reject_reason = "" # type: Optional[str] + reject_reason: Optional[str] = "" # Only specified if it differs from mempool acceptance error. block_reject_reason = "" diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 97f24e1b6e..a0eb213a78 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -76,6 +76,9 @@ class ExampleTest(BitcoinTestFramework): """Override test parameters for your individual test. This method must be overridden and num_nodes must be explicitly set.""" + # By default every test loads a pre-mined chain of 200 blocks from cache. + # Set setup_clean_chain to True to skip this and start from the Genesis + # block. self.setup_clean_chain = True self.num_nodes = 3 # Use self.extra_args to change command-line arguments for the nodes diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 2c6553fbe2..5fcecb4882 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -36,7 +36,6 @@ def expected_messages(filename): class AsmapTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def test_without_asmap_arg(self): diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 603d7f5d3b..1a148f04f4 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -29,9 +29,11 @@ Start three nodes: block 200. node2 will reject block 102 since it's assumed valid, but it isn't buried by at least two weeks' work. """ -import time -from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, @@ -79,24 +81,6 @@ class AssumeValidTest(BitcoinTestFramework): assert not p2p_conn.is_connected break - def assert_blockchain_height(self, node, height): - """Wait until the blockchain is no longer advancing and verify it's reached the expected height.""" - last_height = node.getblock(node.getbestblockhash())['height'] - timeout = 10 - while True: - time.sleep(0.25) - current_height = node.getblock(node.getbestblockhash())['height'] - if current_height != last_height: - last_height = current_height - if timeout < 0: - assert False, "blockchain too short after timeout: %d" % current_height - timeout - 0.25 - continue - elif current_height > height: - assert False, "blockchain too long: %d" % current_height - elif current_height == height: - break - def run_test(self): p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) @@ -177,7 +161,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node0. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p0) - self.assert_blockchain_height(self.nodes[0], 101) + self.wait_until(lambda: self.nodes[0].getblockcount() >= 101) + assert_equal(self.nodes[0].getblockcount(), 101) # Send all blocks to node1. All blocks will be accepted. for i in range(2202): @@ -188,7 +173,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node2. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p2) - self.assert_blockchain_height(self.nodes[2], 101) + self.wait_until(lambda: self.nodes[2].getblockcount() >= 101) + assert_equal(self.nodes[2].getblockcount(), 101) if __name__ == '__main__': diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index b161c71a85..e6a53b52db 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -354,73 +354,75 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): hdkeypath = v17_info["hdkeypath"] pubkey = v17_info["pubkey"] - # Copy the 0.16 wallet to the last Bitcoin Core version and open it: - shutil.copyfile( - os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), - os.path.join(node_master_wallets_dir, "u1_v16") - ) - load_res = node_master.loadwallet("u1_v16") - # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 - assert_equal(load_res['warning'], '') - wallet = node_master.get_wallet_rpc("u1_v16") - info = wallet.getaddressinfo(v16_addr) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it - os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) - shutil.copyfile( - os.path.join(node_master_wallets_dir, "u1_v16"), - os.path.join(node_v16_wallets_dir, "wallets/u1_v16") - ) - self.start_node(-1, extra_args=["-wallet=u1_v16"]) - wallet = node_v16.get_wallet_rpc("u1_v16") - info = wallet.validateaddress(v16_addr) - assert_equal(info, v16_info) - - # Copy the 0.17 wallet to the last Bitcoin Core version and open it: - node_v17.unloadwallet("u1_v17") - shutil.copytree( - os.path.join(node_v17_wallets_dir, "u1_v17"), - os.path.join(node_master_wallets_dir, "u1_v17") - ) - node_master.loadwallet("u1_v17") - wallet = node_master.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it - node_master.unloadwallet("u1_v17") - shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "u1_v17"), - os.path.join(node_v17_wallets_dir, "u1_v17") - ) - node_v17.loadwallet("u1_v17") - wallet = node_v17.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - assert_equal(info, v17_info) - - # Copy the 0.19 wallet to the last Bitcoin Core version and open it: - shutil.copytree( - os.path.join(node_v19_wallets_dir, "w1_v19"), - os.path.join(node_master_wallets_dir, "w1_v19") - ) - node_master.loadwallet("w1_v19") - wallet = node_master.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + if self.is_bdb_compiled(): + # Old wallets are BDB and will only work if BDB is compiled + # Copy the 0.16 wallet to the last Bitcoin Core version and open it: + shutil.copyfile( + os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), + os.path.join(node_master_wallets_dir, "u1_v16") + ) + load_res = node_master.loadwallet("u1_v16") + # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 + assert_equal(load_res['warning'], '') + wallet = node_master.get_wallet_rpc("u1_v16") + info = wallet.getaddressinfo(v16_addr) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it + os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) + shutil.copyfile( + os.path.join(node_master_wallets_dir, "u1_v16"), + os.path.join(node_v16_wallets_dir, "wallets/u1_v16") + ) + self.start_node(-1, extra_args=["-wallet=u1_v16"]) + wallet = node_v16.get_wallet_rpc("u1_v16") + info = wallet.validateaddress(v16_addr) + assert_equal(info, v16_info) - # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it - node_master.unloadwallet("w1_v19") - shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "w1_v19"), - os.path.join(node_v19_wallets_dir, "w1_v19") - ) - node_v19.loadwallet("w1_v19") - wallet = node_v19.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + # Copy the 0.17 wallet to the last Bitcoin Core version and open it: + node_v17.unloadwallet("u1_v17") + shutil.copytree( + os.path.join(node_v17_wallets_dir, "u1_v17"), + os.path.join(node_master_wallets_dir, "u1_v17") + ) + node_master.loadwallet("u1_v17") + wallet = node_master.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: + shutil.copytree( + os.path.join(node_v19_wallets_dir, "w1_v19"), + os.path.join(node_master_wallets_dir, "w1_v19") + ) + node_master.loadwallet("w1_v19") + wallet = node_master.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py new file mode 100755 index 0000000000..d13d191b20 --- /dev/null +++ b/test/functional/feature_blockfilterindex_prune.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test blockfilterindex in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + + +class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] + + def sync_index(self, height): + expected = {'basic block filter index': {'synced': True, 'best_block_height': height}} + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) + + def run_test(self): + self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") + self.sync_index(height=200) + assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) + # Mine two batches of blocks to avoid hitting NODE_NETWORK_LIMITED_MIN_BLOCKS disconnection + self.nodes[0].generate(250) + self.sync_all() + self.nodes[0].generate(250) + self.sync_all() + self.sync_index(height=700) + + self.log.info("prune some blocks") + pruneheight = self.nodes[0].pruneblockchain(400) + assert_equal(pruneheight, 250) + + self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") + assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) + + self.log.info("check if we can access the blockfilter of a pruned block") + assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) + + self.log.info("start node without blockfilterindex") + self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) + + self.log.info("make sure accessing the blockfilters throws an error") + assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) + self.nodes[0].generate(1000) + + self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") + pruneheight_new = self.nodes[0].pruneblockchain(1000) + assert_greater_than(pruneheight_new, pruneheight) + self.stop_node(0) + + self.log.info("make sure we get an init error when starting the node again with block filters") + with self.nodes[0].assert_debug_log(["basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"]): + self.nodes[0].assert_start_raises_init_error(extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) + + self.log.info("make sure the node starts again with the -reindex arg") + self.start_node(0, extra_args = ["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"]) + + +if __name__ == '__main__': + FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index aad255c4a9..b7c2887ee8 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -126,8 +126,13 @@ class BIP65Test(BitcoinTestFramework): # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) + [{ + 'txid': spendtx.hash, + 'wtxid': spendtx.getwtxid(), + 'allowed': False, + 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)', + }], + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 3e28dae4b3..a0bcd9f12a 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -5,8 +5,10 @@ """Test various command line arguments and configuration file parameters.""" import os +import time from test_framework.test_framework import BitcoinTestFramework +from test_framework import util class ConfArgsTest(BitcoinTestFramework): @@ -17,7 +19,7 @@ class ConfArgsTest(BitcoinTestFramework): self.wallet_names = [] def test_config_file_parser(self): - # Assume node is stopped + self.stop_node(0) inc_conf_file_path = os.path.join(self.nodes[0].datadir, 'include.conf') with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: @@ -42,10 +44,11 @@ class ConfArgsTest(BitcoinTestFramework): conf.write("wallet=foo\n") self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on %s network when in [%s] section.' % (self.chain, self.chain)) + main_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_main.conf') + util.write_config(main_conf_file_path, n=0, chain='', extra_config='includeconf={}\n'.format(inc_conf_file_path)) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('regtest=0\n') # mainnet conf.write('acceptnonstdtxn=1\n') - self.nodes[0].assert_start_raises_init_error(expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') + self.nodes[0].assert_start_raises_init_error(extra_args=["-conf={}".format(main_conf_file_path)], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') @@ -86,11 +89,12 @@ class ConfArgsTest(BitcoinTestFramework): ) def test_log_buffer(self): + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0\n']): self.start_node(0, extra_args=['-noconnect=0']) - self.stop_node(0) def test_args_log(self): + self.stop_node(0) self.log.info('Test config args logging') with self.nodes[0].assert_debug_log( expected_msgs=[ @@ -117,39 +121,102 @@ class ConfArgsTest(BitcoinTestFramework): '-rpcuser=secret-rpcuser', '-torpassword=secret-torpassword', ]) - self.stop_node(0) def test_networkactive(self): self.log.info('Test -networkactive option') + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.start_node(0) - self.stop_node(0) + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.start_node(0, extra_args=['-networkactive']) - self.stop_node(0) + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.start_node(0, extra_args=['-networkactive=1']) - self.stop_node(0) + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): self.start_node(0, extra_args=['-networkactive=0']) - self.stop_node(0) + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): self.start_node(0, extra_args=['-nonetworkactive']) - self.stop_node(0) + self.stop_node(0) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): self.start_node(0, extra_args=['-nonetworkactive=1']) + + def test_seed_peers(self): + self.log.info('Test seed peers') + default_data_dir = self.nodes[0].datadir + # Only regtest has no fixed seeds. To avoid connections to random + # nodes, regtest is the only network where it is safe to enable + # -fixedseeds in tests + util.assert_equal(self.nodes[0].getblockchaininfo()['chain'],'regtest') self.stop_node(0) - def run_test(self): + # No peers.dat exists and -dnsseed=1 + # We expect the node will use DNS Seeds, but Regtest mode has 0 DNS seeds + # So after 60 seconds, the node should fallback to fixed seeds (this is a slow test) + assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) + start = int(time.time()) + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "0 addresses found from DNS seeds", + ]): + self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}']) + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Adding fixed seeds as 60 seconds have passed and addrman is empty", + ]): + self.nodes[0].setmocktime(start + 65) self.stop_node(0) + # No peers.dat exists and -dnsseed=0 + # We expect the node will fallback immediately to fixed seeds + assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) + start = time.time() + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "DNS seeding disabled", + "Adding fixed seeds as -dnsseed=0, -addnode is not provided and all -seednode(s) attempted\n", + ]): + self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1']) + assert time.time() - start < 60 + self.stop_node(0) + + # No peers.dat exists and dns seeds are disabled. + # We expect the node will not add fixed seeds when explicitly disabled. + assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) + start = time.time() + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "DNS seeding disabled", + "Fixed seeds are disabled", + ]): + self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=0']) + assert time.time() - start < 60 + self.stop_node(0) + + # No peers.dat exists and -dnsseed=0, but a -addnode is provided + # We expect the node will allow 60 seconds prior to using fixed seeds + assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) + start = int(time.time()) + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "DNS seeding disabled", + ]): + self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1', '-addnode=fakenodeaddr', f'-mocktime={start}']) + with self.nodes[0].assert_debug_log(expected_msgs=[ + "Adding fixed seeds as 60 seconds have passed and addrman is empty", + ]): + self.nodes[0].setmocktime(start + 65) + + def run_test(self): self.test_log_buffer() self.test_args_log() + self.test_seed_peers() self.test_networkactive() self.test_config_file_parser() diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index f9ece244fb..2b56bc78f5 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -49,7 +49,6 @@ from test_framework.util import ( class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 - self.setup_clean_chain = False self.rpc_timeout = 480 self.supports_cli = False diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 3f7efdbded..3b430139b1 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -112,8 +112,13 @@ class BIP66Test(BitcoinTestFramework): # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) + [{ + 'txid': spendtx.hash, + 'wtxid': spendtx.getwtxid(), + 'allowed': False, + 'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)', + }], + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 7de9a589be..2798d11b0a 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -4,6 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" import os +import random +import string from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch @@ -27,11 +29,21 @@ class FilelockTest(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): - self.nodes[0].createwallet(self.default_wallet_name) - wallet_dir = os.path.join(datadir, 'wallets') - self.log.info("Check that we can't start a second bitcoind instance using the same wallet") - expected_msg = "Error: Error initializing wallet database environment" - self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + self.default_wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + def check_wallet_filelock(descriptors): + wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)]) + self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors) + wallet_dir = os.path.join(datadir, 'wallets') + self.log.info("Check that we can't start a second bitcoind instance using the same wallet") + if descriptors: + expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + else: + expected_msg = "Error: Error initializing wallet database environment" + self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + + if self.is_bdb_compiled(): + check_wallet_filelock(False) + if self.is_sqlite_compiled(): + check_wallet_filelock(True) if __name__ == '__main__': FilelockTest().main() diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 6f1a0cd348..f22b7f266a 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -20,7 +20,6 @@ from test_framework.test_framework import BitcoinTestFramework class IncludeConfTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def setup_chain(self): diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index f2313bac13..b068ce612c 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -5,11 +5,11 @@ """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - hex_str_to_bytes, ) # Linux allow all characters other than \x00 @@ -49,6 +49,31 @@ class NotificationsTest(BitcoinTestFramework): super().setup_network() def run_test(self): + if self.is_wallet_compiled(): + # Setup the descriptors to be imported to the wallet + seed = "cTdGmKFWpbvpKQ7ejrdzqYT2hhjyb3GPHnLAK7wdi5Em67YLwSm9" + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + desc_imports = [{ + "desc": descsum_create("wpkh(" + xpriv + "/0/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + },{ + "desc": descsum_create("wpkh(" + xpriv + "/1/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + "internal": True, + }] + # Make the wallets and import the descriptors + # Ensures that node 0 and node 1 share the same wallet for the conflicting transaction tests below. + for i, name in enumerate(self.wallet_names): + self.nodes[i].createwallet(wallet_name=name, descriptors=self.options.descriptors, blank=True, load_on_startup=True) + if self.options.descriptors: + self.nodes[i].importdescriptors(desc_imports) + else: + self.nodes[i].sethdseed(True, seed) + self.log.info("test -blocknotify") block_count = 10 blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) @@ -84,11 +109,10 @@ class NotificationsTest(BitcoinTestFramework): for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) - # Conflicting transactions tests. Give node 0 same wallet seed as - # node 1, generate spends from node 0, and check notifications + # Conflicting transactions tests. + # Generate spends from node 0, and check notifications # triggered by node 1 self.log.info("test -walletnotify with conflicting transactions") - self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) self.nodes[0].rescanblockchain() self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) self.sync_blocks() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index cd5eff9184..8bee43b8ad 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -44,14 +44,15 @@ from test_framework.netutil import test_ipv6_local RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports -# Networks returned by RPC getpeerinfo, defined in src/netbase.cpp::GetNetworkName() -NET_UNROUTABLE = "unroutable" +# Networks returned by RPC getpeerinfo. +NET_UNROUTABLE = "not_publicly_routable" NET_IPV4 = "ipv4" NET_IPV6 = "ipv6" NET_ONION = "onion" +NET_I2P = "i2p" # Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() -NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION}) +NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION, NET_I2P}) class ProxyTest(BitcoinTestFramework): @@ -90,11 +91,15 @@ class ProxyTest(BitcoinTestFramework): self.serv3 = Socks5Server(self.conf3) self.serv3.start() + # We will not try to connect to this. + self.i2p_sam = ('127.0.0.1', 7656) + # Note: proxies are not used to connect to local nodes. This is because the proxy to # use is based on CService.GetNetwork(), which returns NET_UNROUTABLE for localhost. args = [ ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], - ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], + ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr), + '-i2psam=%s:%i' % (self.i2p_sam), '-i2pacceptincoming=0', '-proxyrandomize=0'], ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] ] @@ -199,9 +204,16 @@ class ProxyTest(BitcoinTestFramework): n0 = networks_dict(self.nodes[0].getnetworkinfo()) assert_equal(NETWORKS, n0.keys()) for net in NETWORKS: - assert_equal(n0[net]['proxy'], '%s:%i' % (self.conf1.addr)) - assert_equal(n0[net]['proxy_randomize_credentials'], True) + if net == NET_I2P: + expected_proxy = '' + expected_randomize = False + else: + expected_proxy = '%s:%i' % (self.conf1.addr) + expected_randomize = True + assert_equal(n0[net]['proxy'], expected_proxy) + assert_equal(n0[net]['proxy_randomize_credentials'], expected_randomize) assert_equal(n0['onion']['reachable'], True) + assert_equal(n0['i2p']['reachable'], False) n1 = networks_dict(self.nodes[1].getnetworkinfo()) assert_equal(NETWORKS, n1.keys()) @@ -211,21 +223,36 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['onion']['proxy'], '%s:%i' % (self.conf2.addr)) assert_equal(n1['onion']['proxy_randomize_credentials'], False) assert_equal(n1['onion']['reachable'], True) + assert_equal(n1['i2p']['proxy'], '%s:%i' % (self.i2p_sam)) + assert_equal(n1['i2p']['proxy_randomize_credentials'], False) + assert_equal(n1['i2p']['reachable'], True) n2 = networks_dict(self.nodes[2].getnetworkinfo()) assert_equal(NETWORKS, n2.keys()) for net in NETWORKS: - assert_equal(n2[net]['proxy'], '%s:%i' % (self.conf2.addr)) - assert_equal(n2[net]['proxy_randomize_credentials'], True) + if net == NET_I2P: + expected_proxy = '' + expected_randomize = False + else: + expected_proxy = '%s:%i' % (self.conf2.addr) + expected_randomize = True + assert_equal(n2[net]['proxy'], expected_proxy) + assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) assert_equal(n2['onion']['reachable'], True) + assert_equal(n2['i2p']['reachable'], False) if self.have_ipv6: n3 = networks_dict(self.nodes[3].getnetworkinfo()) assert_equal(NETWORKS, n3.keys()) for net in NETWORKS: - assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) + if net == NET_I2P: + expected_proxy = '' + else: + expected_proxy = '[%s]:%i' % (self.conf3.addr) + assert_equal(n3[net]['proxy'], expected_proxy) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) + assert_equal(n3['i2p']['reachable'], False) if __name__ == '__main__': diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 6ee2b72c11..5027a9828f 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -517,7 +517,6 @@ def add_spender(spenders, *args, **kwargs): def random_checksig_style(pubkey): """Creates a random CHECKSIG* tapscript that would succeed with only the valid signature on witness stack.""" - return bytes(CScript([pubkey, OP_CHECKSIG])) opcode = random.choice([OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKSIGADD]) if (opcode == OP_CHECKSIGVERIFY): ret = CScript([pubkey, opcode, OP_1]) diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py new file mode 100755 index 0000000000..6e6046d84d --- /dev/null +++ b/test/functional/feature_utxo_set_hash.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test UTXO set hash value calculation in gettxoutsetinfo.""" + +import struct + +from test_framework.blocktools import create_transaction +from test_framework.messages import ( + CBlock, + COutPoint, + FromHex, +) +from test_framework.muhash import MuHash3072 +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class UTXOSetHashTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_deterministic_hash_results(self): + self.log.info("Test deterministic UTXO set hash results") + + # These depend on the setup_clean_chain option, the chain loaded from the cache + assert_equal(self.nodes[0].gettxoutsetinfo()['hash_serialized_2'], "b32ec1dda5a53cd025b95387aad344a801825fe46a60ff952ce26528f01d3be8") + assert_equal(self.nodes[0].gettxoutsetinfo("muhash")['muhash'], "dd5ad2a105c2d29495f577245c357409002329b9f4d6182c0af3dc2f462555c8") + + def test_muhash_implementation(self): + self.log.info("Test MuHash implementation consistency") + + node = self.nodes[0] + + # Generate 100 blocks and remove the first since we plan to spend its + # coinbase + block_hashes = node.generate(100) + blocks = list(map(lambda block: FromHex(CBlock(), node.getblock(block, False)), block_hashes)) + spending = blocks.pop(0) + + # Create a spending transaction and mine a block which includes it + tx = create_transaction(node, spending.vtx[0].rehash(), node.getnewaddress(), amount=49) + txid = node.sendrawtransaction(hexstring=tx.serialize_with_witness().hex(), maxfeerate=0) + + tx_block = node.generateblock(output=node.getnewaddress(), transactions=[txid]) + blocks.append(FromHex(CBlock(), node.getblock(tx_block['hash'], False))) + + # Serialize the outputs that should be in the UTXO set and add them to + # a MuHash object + muhash = MuHash3072() + + for height, block in enumerate(blocks): + # The Genesis block coinbase is not part of the UTXO set and we + # spent the first mined block + height += 2 + + for tx in block.vtx: + for n, tx_out in enumerate(tx.vout): + coinbase = 1 if not tx.vin[0].prevout.hash else 0 + + # Skip witness commitment + if (coinbase and n > 0): + continue + + data = COutPoint(int(tx.rehash(), 16), n).serialize() + data += struct.pack("<i", height * 2 + coinbase) + data += tx_out.serialize() + + muhash.insert(data) + + finalized = muhash.digest() + node_muhash = node.gettxoutsetinfo("muhash")['muhash'] + + assert_equal(finalized[::-1].hex(), node_muhash) + + def run_test(self): + self.test_deterministic_hash_results() + self.test_muhash_implementation() + + +if __name__ == '__main__': + UTXOSetHashTest().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 1257dff1ae..2cf0ef2251 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -29,6 +29,8 @@ class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + if self.is_wallet_compiled(): + self.requires_wallet = True def skip_test_if_missing_module(self): self.skip_if_no_cli() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 9b2414cf2d..94e162b748 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -27,28 +27,31 @@ def hash256_reversed(byte_str): class ZMQSubscriber: def __init__(self, socket, topic): - self.sequence = 0 + self.sequence = None # no sequence number received yet self.socket = socket self.topic = topic self.socket.setsockopt(zmq.SUBSCRIBE, self.topic) - def receive(self): + # Receive message from publisher and verify that topic and sequence match + def _receive_from_publisher_and_check(self): topic, body, seq = self.socket.recv_multipart() # Topic should match the subscriber topic. assert_equal(topic, self.topic) # Sequence should be incremental. - assert_equal(struct.unpack('<I', seq)[-1], self.sequence) + received_seq = struct.unpack('<I', seq)[-1] + if self.sequence is None: + self.sequence = received_seq + else: + assert_equal(received_seq, self.sequence) self.sequence += 1 return body + def receive(self): + return self._receive_from_publisher_and_check() + def receive_sequence(self): - topic, body, seq = self.socket.recv_multipart() - # Topic should match the subscriber topic. - assert_equal(topic, self.topic) - # Sequence should be incremental. - assert_equal(struct.unpack('<I', seq)[-1], self.sequence) - self.sequence += 1 + body = self._receive_from_publisher_and_check() hash = body[:32].hex() label = chr(body[32]) mempool_sequence = None if len(body) != 32+1+8 else struct.unpack("<Q", body[32+1:])[0] @@ -59,9 +62,39 @@ class ZMQSubscriber: return (hash, label, mempool_sequence) +class ZMQTestSetupBlock: + """Helper class for setting up a ZMQ test via the "sync up" procedure. + Generates a block on the specified node on instantiation and provides a + method to check whether a ZMQ notification matches, i.e. the event was + caused by this generated block. Assumes that a notification either contains + the generated block's hash, it's (coinbase) transaction id, the raw block or + raw transaction data. + """ + + def __init__(self, node): + self.block_hash = node.generate(1)[0] + coinbase = node.getblock(self.block_hash, 2)['tx'][0] + self.tx_hash = coinbase['txid'] + self.raw_tx = coinbase['hex'] + self.raw_block = node.getblock(self.block_hash, 0) + + def caused_notification(self, notification): + return ( + self.block_hash in notification + or self.tx_hash in notification + or self.raw_block in notification + or self.raw_tx in notification + ) + + class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + if self.is_wallet_compiled(): + self.requires_wallet = True + # This test isn't testing txn relay/timing, so set whitelist on the + # peers for instant txn relay. This speeds up the test run time 2-3x. + self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_py3_zmq() @@ -80,34 +113,65 @@ class ZMQTest (BitcoinTestFramework): self.log.debug("Destroying ZMQ context") self.ctx.destroy(linger=None) + # Restart node with the specified zmq notifications enabled, subscribe to + # all of them and return the corresponding ZMQSubscriber objects. + def setup_zmq_test(self, services, *, recv_timeout=60, sync_blocks=True): + subscribers = [] + for topic, address in services: + socket = self.ctx.socket(zmq.SUB) + subscribers.append(ZMQSubscriber(socket, topic.encode())) + + self.restart_node(0, ["-zmqpub%s=%s" % (topic, address) for topic, address in services] + + self.extra_args[0]) + + for i, sub in enumerate(subscribers): + sub.socket.connect(services[i][1]) + + # Ensure that all zmq publisher notification interfaces are ready by + # running the following "sync up" procedure: + # 1. Generate a block on the node + # 2. Try to receive the corresponding notification on all subscribers + # 3. If all subscribers get the message within the timeout (1 second), + # we are done, otherwise repeat starting from step 1 + for sub in subscribers: + sub.socket.set(zmq.RCVTIMEO, 1000) + while True: + test_block = ZMQTestSetupBlock(self.nodes[0]) + recv_failed = False + for sub in subscribers: + try: + while not test_block.caused_notification(sub.receive().hex()): + self.log.debug("Ignoring sync-up notification for previously generated block.") + except zmq.error.Again: + self.log.debug("Didn't receive sync-up notification, trying again.") + recv_failed = True + if not recv_failed: + self.log.debug("ZMQ sync-up completed, all subscribers are ready.") + break + + # set subscriber's desired timeout for the test + for sub in subscribers: + sub.socket.set(zmq.RCVTIMEO, recv_timeout*1000) + + self.connect_nodes(0, 1) + if sync_blocks: + self.sync_blocks() + + return subscribers + def test_basic(self): # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = 'tcp://127.0.0.1:28332' - sockets = [] - subs = [] - services = [b"hashblock", b"hashtx", b"rawblock", b"rawtx"] - for service in services: - sockets.append(self.ctx.socket(zmq.SUB)) - sockets[-1].set(zmq.RCVTIMEO, 60000) - subs.append(ZMQSubscriber(sockets[-1], service)) - - # Subscribe to all available topics. + subs = self.setup_zmq_test([(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]]) + hashblock = subs[0] hashtx = subs[1] rawblock = subs[2] rawtx = subs[3] - self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx, rawblock, rawtx]]) - self.connect_nodes(0, 1) - for socket in sockets: - socket.connect(address) - - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) - num_blocks = 5 self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) genhashes = self.nodes[0].generatetoaddress(num_blocks, ADDRESS_BCRT1_UNSPENDABLE) @@ -174,25 +238,11 @@ class ZMQTest (BitcoinTestFramework): address = 'tcp://127.0.0.1:28333' - services = [b"hashblock", b"hashtx"] - sockets = [] - subs = [] - for service in services: - sockets.append(self.ctx.socket(zmq.SUB)) - # 2 second timeout to check end of notifications - sockets[-1].set(zmq.RCVTIMEO, 2000) - subs.append(ZMQSubscriber(sockets[-1], service)) - - # Subscribe to all available topics. - hashblock = subs[0] - hashtx = subs[1] - # Should only notify the tip if a reorg occurs - self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx]]) - for socket in sockets: - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + hashblock, hashtx = self.setup_zmq_test( + [(topic, address) for topic in ["hashblock", "hashtx"]], + recv_timeout=2) # 2 second timeout to check end of notifications + self.disconnect_nodes(0, 1) # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) @@ -240,15 +290,8 @@ class ZMQTest (BitcoinTestFramework): <32-byte hash>A<8-byte LE uint> : Transactionhash added mempool """ self.log.info("Testing 'sequence' publisher") - address = 'tcp://127.0.0.1:28333' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - seq = ZMQSubscriber(socket, b'sequence') - - self.restart_node(0, ['-zmqpub%s=%s' % (seq.topic.decode(), address)]) - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) + self.disconnect_nodes(0, 1) # Mempool sequence number starts at 1 seq_num = 1 @@ -323,7 +366,7 @@ class ZMQTest (BitcoinTestFramework): block_count = self.nodes[0].getblockcount() best_hash = self.nodes[0].getbestblockhash() self.nodes[0].invalidateblock(best_hash) - sleep(2) # Bit of room to make sure transaction things happened + sleep(2) # Bit of room to make sure transaction things happened # Make sure getrawmempool mempool_sequence results aren't "queued" but immediately reflective # of the time they were gathered. @@ -372,8 +415,8 @@ class ZMQTest (BitcoinTestFramework): assert_equal(label, "A") # More transactions to be simply mined for i in range(len(more_tx)): - assert_equal((more_tx[i], "A", mempool_seq), seq.receive_sequence()) - mempool_seq += 1 + assert_equal((more_tx[i], "A", mempool_seq), seq.receive_sequence()) + mempool_seq += 1 # Bumped by rbf assert_equal((orig_txid, "R", mempool_seq), seq.receive_sequence()) mempool_seq += 1 @@ -388,7 +431,7 @@ class ZMQTest (BitcoinTestFramework): assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) mempool_seq += 1 self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() # want to make sure we didn't break "consensus" for other tests + self.sync_all() # want to make sure we didn't break "consensus" for other tests def test_mempool_sync(self): """ @@ -399,16 +442,7 @@ class ZMQTest (BitcoinTestFramework): return self.log.info("Testing 'mempool sync' usage of sequence notifier") - address = 'tcp://127.0.0.1:28333' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - seq = ZMQSubscriber(socket, b'sequence') - - self.restart_node(0, ['-zmqpub%s=%s' % (seq.topic.decode(), address)]) - self.connect_nodes(0, 1) - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) # In-memory counter, should always start at 1 next_mempool_seq = self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] @@ -508,26 +542,20 @@ class ZMQTest (BitcoinTestFramework): def test_multiple_interfaces(self): # Set up two subscribers with different addresses - subscribers = [] - for i in range(2): - address = 'tcp://127.0.0.1:%d' % (28334 + i) - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - hashblock = ZMQSubscriber(socket, b"hashblock") - socket.connect(address) - subscribers.append({'address': address, 'hashblock': hashblock}) - - self.restart_node(0, ['-zmqpub%s=%s' % (subscriber['hashblock'].topic.decode(), subscriber['address']) for subscriber in subscribers]) - - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + # (note that after the reorg test, syncing would fail due to different + # chain lengths on node0 and node1; for this test we only need node0, so + # we can disable syncing blocks on the setup) + subscribers = self.setup_zmq_test([ + ("hashblock", "tcp://127.0.0.1:28334"), + ("hashblock", "tcp://127.0.0.1:28335"), + ], sync_blocks=False) # Generate 1 block in nodes[0] and receive all notifications self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) # Should receive the same block hash on both subscribers - assert_equal(self.nodes[0].getbestblockhash(), subscribers[0]['hashblock'].receive().hex()) - assert_equal(self.nodes[0].getbestblockhash(), subscribers[1]['hashblock'].receive().hex()) + assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) + assert_equal(self.nodes[0].getbestblockhash(), subscribers[1].receive().hex()) if __name__ == '__main__': ZMQTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index fc4c775668..c4002f524a 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -51,6 +51,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def check_mempool_result(self, result_expected, *args, **kwargs): """Wrapper to check result of testmempoolaccept on node_0's mempool""" result_test = self.nodes[0].testmempoolaccept(*args, **kwargs) + for r in result_test: + r.pop('wtxid') # Skip check for now assert_equal(result_expected, result_test) assert_equal(self.nodes[0].getmempoolinfo()['size'], self.mempool_size) # Must not change mempool state diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 70cd4ebb3b..752b925b92 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -69,6 +69,8 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + total_fee_old = self.nodes[0].getmempoolinfo()['total_fee'] + self.log.debug("Prioritize a transaction on node0") fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'], fees['modified']) @@ -76,6 +78,10 @@ class MempoolPersistTest(BitcoinTestFramework): fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.info('Check the total base fee is unchanged after prioritisetransaction') + assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee']) + assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items())) + tx_creation_time = self.nodes[0].getmempoolentry(txid=last_txid)['time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py new file mode 100755 index 0000000000..676d0a0a4d --- /dev/null +++ b/test/functional/mocks/signer.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import os +import sys +import argparse +import json + +def perform_pre_checks(): + mock_result_path = os.path.join(os.getcwd(), "mock_result") + if(os.path.isfile(mock_result_path)): + with open(mock_result_path, "r", encoding="utf8") as f: + mock_result = f.read() + if mock_result[0]: + sys.stdout.write(mock_result[2:]) + sys.exit(int(mock_result[0])) + +def enumerate(args): + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + +def getdescriptors(args): + xpub = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" + + sys.stdout.write(json.dumps({ + "receive": [ + "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", + "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs" + ], + "internal": [ + "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", + "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg" + ] + })) + + +def displayaddress(args): + # Several descriptor formats are acceptable, so allowing for potential + # changes to InferDescriptor: + if args.fingerprint != "00000001": + return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) + + expected_desc = [ + "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r" + ] + if args.desc not in expected_desc: + return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) + + return sys.stdout.write(json.dumps({"address": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g"})) + +def signtx(args): + if args.fingerprint != "00000001": + return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) + + with open(os.path.join(os.getcwd(), "mock_psbt"), "r", encoding="utf8") as f: + mock_psbt = f.read() + + if args.fingerprint == "00000001" : + sys.stdout.write(json.dumps({ + "psbt": mock_psbt, + "complete": True + })) + else: + sys.stdout.write(json.dumps({"psbt": args.psbt})) + +parser = argparse.ArgumentParser(prog='./signer.py', description='External signer mock') +parser.add_argument('--fingerprint') +parser.add_argument('--chain', default='main') +parser.add_argument('--stdin', action='store_true') + +subparsers = parser.add_subparsers(description='Commands', dest='command') +subparsers.required = True + +parser_enumerate = subparsers.add_parser('enumerate', help='list available signers') +parser_enumerate.set_defaults(func=enumerate) + +parser_getdescriptors = subparsers.add_parser('getdescriptors') +parser_getdescriptors.set_defaults(func=getdescriptors) +parser_getdescriptors.add_argument('--account', metavar='account') + +parser_displayaddress = subparsers.add_parser('displayaddress', help='display address on signer') +parser_displayaddress.add_argument('--desc', metavar='desc') +parser_displayaddress.set_defaults(func=displayaddress) + +parser_signtx = subparsers.add_parser('signtx') +parser_signtx.add_argument('psbt', metavar='psbt') + +parser_signtx.set_defaults(func=signtx) + +if not sys.stdin.isatty(): + buffer = sys.stdin.read() + if buffer and buffer.rstrip() != "": + sys.argv.extend(buffer.rstrip().split(" ")) + +args = parser.parse_args() + +perform_pre_checks() + +args.func(args) diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py new file mode 100755 index 0000000000..a04ba5db2d --- /dev/null +++ b/test/functional/p2p_add_connections.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test add_outbound_p2p_connection test framework functionality""" + +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +def check_node_connections(*, node, num_in, num_out): + info = node.getnetworkinfo() + assert_equal(info["connections_in"], num_in) + assert_equal(info["connections_out"], num_out) + + +class P2PAddConnections(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def setup_network(self): + self.setup_nodes() + # Don't connect the nodes + + def run_test(self): + self.log.info("Add 8 outbounds to node 0") + for i in range(8): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="outbound-full-relay") + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # set p2p_idx based on the outbound connections already open to the + # node, so add 8 to account for the previous full-relay connections + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 8, connection_type="block-relay-only") + + self.log.info("Add 2 block-relay-only connections to node 1") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + self.nodes[1].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="block-relay-only") + + self.log.info("Add 5 inbound connections to node 1") + for i in range(5): + self.log.info(f"inbound: {i}") + self.nodes[1].add_p2p_connection(P2PInterface()) + + self.log.info("Add 8 outbounds to node 1") + for i in range(8): + self.log.info(f"outbound: {i}") + # bump p2p_idx to account for the 2 existing outbounds on node 1 + self.nodes[1].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 2) + + self.log.info("Check the connections opened as expected") + check_node_connections(node=self.nodes[0], num_in=0, num_out=10) + check_node_connections(node=self.nodes[1], num_in=5, num_out=10) + + self.log.info("Disconnect p2p connections & try to re-open") + self.nodes[0].disconnect_p2ps() + check_node_connections(node=self.nodes[0], num_in=0, num_out=0) + + self.log.info("Add 8 outbounds to node 0") + for i in range(8): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i) + check_node_connections(node=self.nodes[0], num_in=0, num_out=8) + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # bump p2p_idx to account for the 8 existing outbounds on node 0 + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 8, connection_type="block-relay-only") + check_node_connections(node=self.nodes[0], num_in=0, num_out=10) + + self.log.info("Restart node 0 and try to reconnect to p2ps") + self.restart_node(0) + + self.log.info("Add 4 outbounds to node 0") + for i in range(4): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i) + check_node_connections(node=self.nodes[0], num_in=0, num_out=4) + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # bump p2p_idx to account for the 4 existing outbounds on node 0 + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 4, connection_type="block-relay-only") + check_node_connections(node=self.nodes[0], num_in=0, num_out=6) + + check_node_connections(node=self.nodes[1], num_in=5, num_out=10) + + +if __name__ == '__main__': + P2PAddConnections().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 91fbd722cf..69821763bd 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -46,7 +46,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 73642a37b3..6584efae79 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -2,62 +2,46 @@ # Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test p2p blocksonly""" +"""Test p2p blocksonly mode & block-relay-only connections.""" -from test_framework.messages import msg_tx, CTransaction, FromHex -from test_framework.p2p import P2PInterface +import time + +from test_framework.blocktools import create_transaction +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [["-blocksonly"]] + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + def run_test(self): - block_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) - - self.log.info('Check that txs from p2p are rejected and result in disconnect') - prevtx = self.nodes[0].getblock(self.nodes[0].getblockhash(1), 2)['tx'][0] - rawtx = self.nodes[0].createrawtransaction( - inputs=[{ - 'txid': prevtx['txid'], - 'vout': 0 - }], - outputs=[{ - self.nodes[0].get_deterministic_priv_key().address: 50 - 0.00125 - }], - ) - sigtx = self.nodes[0].signrawtransactionwithkey( - hexstring=rawtx, - privkeys=[self.nodes[0].get_deterministic_priv_key().key], - prevtxs=[{ - 'txid': prevtx['txid'], - 'vout': 0, - 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], - }], - )['hex'] + self.blocksonly_mode_tests() + self.blocks_relay_conn_tests() + + def blocksonly_mode_tests(self): + self.log.info("Tests with node running in -blocksonly mode") assert_equal(self.nodes[0].getnetworkinfo()['localrelay'], False) - with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): - block_relay_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) - block_relay_peer.wait_for_disconnect() - assert_equal(self.nodes[0].getmempoolinfo()['size'], 0) - # Remove the disconnected peer and add a new one. - del self.nodes[0].p2ps[0] - tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + self.nodes[0].add_p2p_connection(P2PInterface()) + tx, txid, tx_hex = self.check_p2p_tx_violation() self.log.info('Check that txs from rpc are not rejected and relayed to other peers') + tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], True) - txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + + assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(txid)]): - self.nodes[0].sendrawtransaction(sigtx) + self.nodes[0].sendrawtransaction(tx_hex) tx_relay_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info('Check that txs from peers with relay-permission are not rejected and relayed to others') self.log.info("Restarting node 0 with relay permission and blocksonly") self.restart_node(0, ["-persistmempool=0", "-whitelist=relay@127.0.0.1", "-blocksonly"]) assert_equal(self.nodes[0].getrawmempool(), []) @@ -67,8 +51,7 @@ class P2PBlocksOnly(BitcoinTestFramework): assert_equal(peer_1_info['permissions'], ['relay']) peer_2_info = self.nodes[0].getpeerinfo()[1] assert_equal(peer_2_info['permissions'], ['relay']) - assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) - txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) self.log.info('Check that the tx from first_peer with relay-permission is relayed to others (ie.second_peer)') with self.nodes[0].assert_debug_log(["received getdata"]): @@ -78,13 +61,58 @@ class P2PBlocksOnly(BitcoinTestFramework): # But if, for some reason, first_peer decides to relay transactions to us anyway, we should relay them to # second_peer since we gave relay permission to first_peer. # See https://github.com/bitcoin/bitcoin/issues/19943 for details. - first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) + first_peer.send_message(msg_tx(tx)) self.log.info('Check that the peer with relay-permission is still connected after sending the transaction') assert_equal(first_peer.is_connected, True) second_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) self.log.info("Relay-permission peer's transaction is accepted and relayed") + self.nodes[0].disconnect_p2ps() + self.nodes[0].generate(1) + + def blocks_relay_conn_tests(self): + self.log.info('Tests with node in normal mode with block-relay-only connections') + self.restart_node(0, ["-noblocksonly"]) # disables blocks only mode + assert_equal(self.nodes[0].getnetworkinfo()['localrelay'], True) + + # Ensure we disconnect if a block-relay-only connection sends us a transaction + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="block-relay-only") + assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], False) + _, txid, tx_hex = self.check_p2p_tx_violation(index=2) + + self.log.info("Check that txs from RPC are not sent to blockrelay connection") + conn = self.nodes[0].add_outbound_p2p_connection(P2PTxInvStore(), p2p_idx=1, connection_type="block-relay-only") + + self.nodes[0].sendrawtransaction(tx_hex) + + # Bump time forward to ensure nNextInvSend timer pops + self.nodes[0].setmocktime(int(time.time()) + 60) + + # Calling sync_with_ping twice requires that the node calls + # `ProcessMessage` twice, and thus ensures `SendMessages` must have + # been called at least once + conn.sync_with_ping() + conn.sync_with_ping() + assert(int(txid, 16) not in conn.get_invs()) + + def check_p2p_tx_violation(self, index=1): + self.log.info('Check that txs from P2P are rejected and result in disconnect') + input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(index), 2)['tx'][0]['txid'] + tx = create_transaction(self.nodes[0], input_txid, self.nodes[0].getnewaddress(), amount=(50 - 0.001)) + txid = tx.rehash() + tx_hex = tx.serialize().hex() + + with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): + self.nodes[0].p2ps[0].send_message(msg_tx(tx)) + self.nodes[0].p2ps[0].wait_for_disconnect() + assert_equal(self.nodes[0].getmempoolinfo()['size'], 0) + + # Remove the disconnected peer + del self.nodes[0].p2ps[0] + + return tx, txid, tx_hex + if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 642a217047..8f64419138 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -19,7 +19,13 @@ from test_framework.messages import ( msg_mempool, msg_version, ) -from test_framework.p2p import P2PInterface, p2p_lock +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, + P2P_SUBVERSION, + P2P_VERSION, + p2p_lock, +) from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -81,7 +87,6 @@ class P2PBloomFilter(P2PInterface): class FilterTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [[ '-peerbloomfilters', @@ -217,9 +222,12 @@ class FilterTest(BitcoinTestFramework): self.log.info('Test BIP 37 for a node with fRelay = False') # Add peer but do not send version yet filter_peer_without_nrelay = self.nodes[0].add_p2p_connection(P2PBloomFilter(), send_version=False, wait_for_verack=False) - # Send version with fRelay=False + # Send version with relay=False version_without_fRelay = msg_version() - version_without_fRelay.nRelay = 0 + version_without_fRelay.nVersion = P2P_VERSION + version_without_fRelay.strSubVer = P2P_SUBVERSION + version_without_fRelay.nServices = P2P_SERVICES + version_without_fRelay.relay = 0 filter_peer_without_nrelay.send_message(version_without_fRelay) filter_peer_without_nrelay.wait_for_verack() assert not self.nodes[0].getpeerinfo()[0]['relaytxes'] diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index 2b75ad5175..d375af6fe1 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -41,7 +41,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index e4fc9fd178..f884cf90ff 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -13,7 +13,6 @@ from test_framework.test_framework import BitcoinTestFramework class InvalidLocatorTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = False def run_test(self): node = self.nodes[0] # convenience reference to the node diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index ca8bf908a9..12b8b7baff 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -17,7 +17,12 @@ from test_framework.messages import ( msg_ping, msg_version, ) -from test_framework.p2p import P2PInterface +from test_framework.p2p import ( + P2PInterface, + P2P_SUBVERSION, + P2P_SERVICES, + P2P_VERSION_RELAY, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -125,12 +130,15 @@ class P2PLeakTest(BitcoinTestFramework): assert_equal(ver.addrFrom.port, 0) assert_equal(ver.addrFrom.ip, '0.0.0.0') assert_equal(ver.nStartingHeight, 201) - assert_equal(ver.nRelay, 1) + assert_equal(ver.relay, 1) self.log.info('Check that old peers are disconnected') p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 + old_version_msg.strSubVer = P2P_SUBVERSION + old_version_msg.nServices = P2P_SERVICES + old_version_msg.relay = P2P_VERSION_RELAY with self.nodes[0].assert_debug_log(['peer=3 using obsolete version 31799; disconnecting']): p2p_old_peer.send_message(old_version_msg) p2p_old_peer.wait_for_disconnect() diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py new file mode 100755 index 0000000000..080b2d93ad --- /dev/null +++ b/test/functional/p2p_message_capture.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test per-peer message capture capability. + +Additionally, the output of contrib/message-capture/message-capture-parser.py should be verified manually. +""" + +import glob +from io import BytesIO +import os + +from test_framework.p2p import P2PDataStore, MESSAGEMAP +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +def mini_parser(dat_file): + """Parse a data file created by CaptureMessage. + + From the data file we'll only check the structure. + + We won't care about things like: + - Deserializing the payload of the message + - This is managed by the deserialize methods in test_framework.messages + - The order of the messages + - There's no reason why we can't, say, change the order of the messages in the handshake + - Message Type + - We can add new message types + + We're ignoring these because they're simply too brittle to test here. + """ + with open(dat_file, 'rb') as f_in: + # This should have at least one message in it + assert(os.fstat(f_in.fileno()).st_size >= TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + while True: + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + tmp_header.read(TIME_SIZE) # skip the timestamp field + raw_msgtype = tmp_header.read(MSGTYPE_SIZE) + msgtype: bytes = raw_msgtype.split(b'\x00', 1)[0] + remainder = raw_msgtype.split(b'\x00', 1)[1] + assert(len(msgtype) > 0) + assert(msgtype in MESSAGEMAP) + assert(len(remainder) == 0 or not remainder.decode().isprintable()) + length: int = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") + data = f_in.read(length) + assert_equal(len(data), length) + + + +class MessageCaptureTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-capturemessages"]] + self.setup_clean_chain = True + + def run_test(self): + capturedir = os.path.join(self.nodes[0].datadir, "regtest/message_capture") + # Connect a node so that the handshake occurs + self.nodes[0].add_p2p_connection(P2PDataStore()) + self.nodes[0].disconnect_p2ps() + recv_file = glob.glob(os.path.join(capturedir, "*/msgs_recv.dat"))[0] + mini_parser(recv_file) + sent_file = glob.glob(os.path.join(capturedir, "*/msgs_sent.dat"))[0] + mini_parser(sent_file) + + +if __name__ == '__main__': + MessageCaptureTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index a9d8b12d70..54891b07e1 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -686,13 +686,35 @@ class SegWitTest(BitcoinTestFramework): if not self.segwit_active: # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00001000')}}]) + assert_equal( + self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + [{ + 'txid': tx3.hash, + 'wtxid': tx3.getwtxid(), + 'allowed': True, + 'vsize': tx3.get_vsize(), + 'fees': { + 'base': Decimal('0.00001000'), + }, + }], + ) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00011000')}}]) + assert_equal( + self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + [{ + 'txid': tx3.hash, + 'wtxid': tx3.getwtxid(), + 'allowed': True, + 'vsize': tx3.get_vsize(), + 'fees': { + 'base': Decimal('0.00011000'), + }, + }], + ) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 47832b04bf..a7e240dcfa 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -74,9 +74,9 @@ class TimeoutsTest(BitcoinTestFramework): no_version_node.send_message(msg_ping()) expected_timeout_logs = [ - "version handshake timeout from 0", - "socket no message in first 3 seconds, 1 0 from 1", - "socket no message in first 3 seconds, 0 0 from 2", + "version handshake timeout peer=0", + "socket no message in first 3 seconds, 1 0 peer=1", + "socket no message in first 3 seconds, 0 0 peer=2", ] with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 8a751c6b54..4bf96cb0e6 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -56,7 +56,6 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def test_tx_requests(self): diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 99be6b7b8e..e090030205 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -23,6 +23,7 @@ import http.client import os import subprocess +from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE from test_framework.blocktools import ( create_block, create_coinbase, @@ -71,11 +72,10 @@ class BlockchainTest(BitcoinTestFramework): def mine_chain(self): self.log.info('Create some old blocks') - address = self.nodes[0].get_deterministic_priv_key().address for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): # ten-minute steps from genesis block time self.nodes[0].setmocktime(t) - self.nodes[0].generatetoaddress(1, address) + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_P2WSH_OP_TRUE) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) def _test_getblockchaininfo(self): @@ -227,7 +227,7 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res['transactions'], 200) assert_equal(res['height'], 200) assert_equal(res['txouts'], 200) - assert_equal(res['bogosize'], 15000), + assert_equal(res['bogosize'], 16800), assert_equal(res['bestblock'], node.getblockhash(200)) size = res['disk_size'] assert size > 6400 @@ -268,6 +268,18 @@ class BlockchainTest(BitcoinTestFramework): res5 = node.gettxoutsetinfo(hash_type='none') assert 'hash_serialized_2' not in res5 + # hash_type muhash should return a different UTXO set hash. + res6 = node.gettxoutsetinfo(hash_type='muhash') + assert 'muhash' in res6 + assert(res['hash_serialized_2'] != res6['muhash']) + + # muhash should not be included in gettxoutset unless requested. + for r in [res, res2, res3, res4, res5]: + assert 'muhash' not in r + + # Unknown hash_type raises an error + assert_raises_rpc_error(-8, "foohash is not a valid hash_type", node.gettxoutsetinfo, "foohash") + def _test_getblockheader(self): node = self.nodes[0] @@ -304,6 +316,9 @@ class BlockchainTest(BitcoinTestFramework): header.calc_sha256() assert_equal(header.hash, besthash) + assert 'previousblockhash' not in node.getblockheader(node.getblockhash(0)) + assert 'nextblockhash' not in node.getblockheader(node.getbestblockhash()) + def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 @@ -317,12 +332,12 @@ class BlockchainTest(BitcoinTestFramework): def _test_stopatheight(self): assert_equal(self.nodes[0].getblockcount(), 200) - self.nodes[0].generatetoaddress(6, self.nodes[0].get_deterministic_priv_key().address) + self.nodes[0].generatetoaddress(6, ADDRESS_BCRT1_P2WSH_OP_TRUE) assert_equal(self.nodes[0].getblockcount(), 206) self.log.debug('Node should not stop at this height') assert_raises(subprocess.TimeoutExpired, lambda: self.nodes[0].process.wait(timeout=3)) try: - self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address) + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_P2WSH_OP_TRUE) except (ConnectionError, http.client.BadStatusLine): pass # The node already shut down before response self.log.debug('Node should stop at this height...') @@ -372,8 +387,7 @@ class BlockchainTest(BitcoinTestFramework): node = self.nodes[0] miniwallet = MiniWallet(node) - miniwallet.generate(5) - node.generate(100) + miniwallet.scan_blocks(num=5) fee_per_byte = Decimal('0.00000010') fee_per_kb = 1000 * fee_per_byte @@ -408,6 +422,9 @@ class BlockchainTest(BitcoinTestFramework): # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') + assert 'previousblockhash' not in node.getblock(node.getblockhash(0)) + assert 'nextblockhash' not in node.getblock(node.getbestblockhash()) + if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 209d1182d7..1af79b9f7c 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error, find_vout_for_address class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): @@ -24,37 +23,7 @@ class DeprecatedRpcTest(BitcoinTestFramework): # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) # self.nodes[1].generate(1) - if self.is_wallet_compiled(): - self.log.info("Test bumpfee RPC") - self.nodes[0].generate(101) - self.nodes[0].createwallet(wallet_name='nopriv', disable_private_keys=True) - noprivs0 = self.nodes[0].get_wallet_rpc('nopriv') - w0 = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - self.nodes[1].createwallet(wallet_name='nopriv', disable_private_keys=True) - noprivs1 = self.nodes[1].get_wallet_rpc('nopriv') - - address = w0.getnewaddress() - desc = w0.getaddressinfo(address)['desc'] - change_addr = w0.getrawchangeaddress() - change_desc = w0.getaddressinfo(change_addr)['desc'] - txid = w0.sendtoaddress(address=address, amount=10) - vout = find_vout_for_address(w0, txid, address) - self.nodes[0].generate(1) - rawtx = w0.createrawtransaction([{'txid': txid, 'vout': vout}], {w0.getnewaddress(): 5}, 0, True) - rawtx = w0.fundrawtransaction(rawtx, {'changeAddress': change_addr}) - signed_tx = w0.signrawtransactionwithwallet(rawtx['hex'])['hex'] - - noprivs0.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) - noprivs1.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) - - txid = w0.sendrawtransaction(signed_tx) - self.sync_all() - - assert_raises_rpc_error(-32, 'Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22', noprivs0.bumpfee, txid) - bumped_psbt = noprivs1.bumpfee(txid) - assert 'psbt' in bumped_psbt - else: - self.log.info("No tested deprecated RPC methods") + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 81862ac69e..51b7efb4c3 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -14,7 +14,6 @@ from test_framework.util import assert_raises_rpc_error class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index c83157a843..de21f43747 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -7,7 +7,39 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +from collections import defaultdict import os +import re + + +def parse_string(s): + assert s[0] == '"' + assert s[-1] == '"' + return s[1:-1] + + +def process_mapping(fname): + """Find and parse conversion table in implementation file `fname`.""" + cmds = [] + in_rpcs = False + with open(fname, "r", encoding="utf8") as f: + for line in f: + line = line.rstrip() + if not in_rpcs: + if line == 'static const CRPCConvertParam vRPCConvertParams[] =': + in_rpcs = True + else: + if line.startswith('};'): + in_rpcs = False + elif '{' in line and '"' in line: + m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) + assert m, 'No match to table expression: %s' % line + name = parse_string(m.group(1)) + idx = int(m.group(2)) + argname = parse_string(m.group(3)) + cmds.append((name, idx, argname)) + assert not in_rpcs and cmds + return cmds class HelpRpcTest(BitcoinTestFramework): @@ -16,11 +48,43 @@ class HelpRpcTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + self.test_client_conversion_table() self.test_categories() self.dump_help() if self.is_wallet_compiled(): self.wallet_help() + def test_client_conversion_table(self): + file_conversion_table = os.path.join(self.config["environment"]["SRCDIR"], 'src', 'rpc', 'client.cpp') + mapping_client = process_mapping(file_conversion_table) + # Ignore echojson in client table + mapping_client = [m for m in mapping_client if m[0] != 'echojson'] + + mapping_server = self.nodes[0].help("dump_all_command_conversions") + # Filter all RPCs whether they need conversion + mapping_server_conversion = [tuple(m[:3]) for m in mapping_server if not m[3]] + + # Only check if all RPC methods have been compiled (i.e. wallet is enabled) + if self.is_wallet_compiled() and sorted(mapping_client) != sorted(mapping_server_conversion): + raise AssertionError("RPC client conversion table ({}) and RPC server named arguments mismatch!\n{}".format( + file_conversion_table, + set(mapping_client).symmetric_difference(mapping_server_conversion), + )) + + # Check for conversion difference by argument name. + # It is preferable for API consistency that arguments with the same name + # have the same conversion, so bin by argument name. + all_methods_by_argname = defaultdict(list) + converts_by_argname = defaultdict(list) + for m in mapping_server: + all_methods_by_argname[m[2]].append(m[0]) + converts_by_argname[m[2]].append(m[3]) + + for argname, convert in converts_by_argname.items(): + if all(convert) != any(convert): + # Only allow dummy to fail consistency check + assert argname == 'dummy', ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) + def test_categories(self): node = self.nodes[0] @@ -41,10 +105,13 @@ class HelpRpcTest(BitcoinTestFramework): if self.is_wallet_compiled(): components.append('Wallet') + if self.is_external_signer_compiled(): + components.append('Signer') + if self.is_zmq_compiled(): components.append('Zmq') - assert_equal(titles, components) + assert_equal(titles, sorted(components)) def dump_help(self): dump_dir = os.path.join(self.options.tmpdir, 'rpc_help_dump') diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py new file mode 100755 index 0000000000..469d6bdb05 --- /dev/null +++ b/test/functional/rpc_invalid_address_message.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test error messages for 'getaddressinfo' and 'validateaddress' RPC commands.""" + +from test_framework.test_framework import BitcoinTestFramework + +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +BECH32_VALID = 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv' +BECH32_INVALID_SIZE = 'bcrt1sqqpl9r5c' +BECH32_INVALID_PREFIX = 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4' + +BASE58_VALID = 'mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJRfn' +BASE58_INVALID_PREFIX = '17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem' + +INVALID_ADDRESS = 'asfah14i8fajz0123f' + +class InvalidAddressErrorMessageTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_validateaddress(self): + node = self.nodes[0] + + # Bech32 + info = node.validateaddress(BECH32_INVALID_SIZE) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid Bech32 address data size') + + info = node.validateaddress(BECH32_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Bech32 address') + + info = node.validateaddress(BECH32_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Base58 + info = node.validateaddress(BASE58_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Base58-encoded address') + + info = node.validateaddress(BASE58_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Invalid address format + info = node.validateaddress(INVALID_ADDRESS) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid address format') + + def test_getaddressinfo(self): + node = self.nodes[0] + + assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) + + assert_raises_rpc_error(-5, "Invalid prefix for Bech32 address", node.getaddressinfo, BECH32_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid address format", node.getaddressinfo, INVALID_ADDRESS) + + def run_test(self): + self.test_validateaddress() + self.test_getaddressinfo() + + +if __name__ == '__main__': + InvalidAddressErrorMessageTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index de0b7f303f..9adb32c3c5 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -25,6 +25,7 @@ from test_framework.util import ( assert_raises_rpc_error, p2p_port, ) +from test_framework.wallet import MiniWallet def assert_net_servicesnames(servicesflag, servicenames): @@ -48,6 +49,9 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + # We need miniwallet to make a transaction + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.generate(1) # Get out of IBD for the minfeefilter and getpeerinfo tests. self.nodes[0].generate(101) @@ -74,8 +78,7 @@ class NetTest(BitcoinTestFramework): def test_getpeerinfo(self): self.log.info("Test getpeerinfo") # Create a few getpeerinfo last_block/last_transaction values. - if self.is_wallet_compiled(): - self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results self.nodes[1].generate(1) self.sync_all() time_now = int(time.time()) @@ -101,6 +104,9 @@ class NetTest(BitcoinTestFramework): assert_equal(peer_info[1][0]['connection_type'], 'manual') assert_equal(peer_info[1][1]['connection_type'], 'inbound') + # Check dynamically generated networks list in getpeerinfo help output. + assert "(ipv4, ipv6, onion, i2p, not_publicly_routable)" in self.nodes[0].help("getpeerinfo") + def test_getnettotals(self): self.log.info("Test getnettotals") # Test getnettotals and getpeerinfo by doing a ping. The bytes @@ -149,6 +155,9 @@ class NetTest(BitcoinTestFramework): for info in network_info: assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) + # Check dynamically generated networks list in getnetworkinfo help output. + assert "(ipv4, ipv6, onion, i2p)" in self.nodes[0].help("getnetworkinfo") + def test_getaddednodeinfo(self): self.log.info("Test getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b364077a9a..ed6abaed78 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -24,7 +24,6 @@ MAX_BIP125_RBF_SEQUENCE = 0xfffffffd class PSBTTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 3 self.extra_args = [ ["-walletrbf=1"], diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index 523fdc9f63..fd5f8aa098 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -15,6 +15,9 @@ class SetBanTests(BitcoinTestFramework): self.setup_clean_chain = True self.extra_args = [[],[]] + def is_banned(self, node, addr): + return any(e['address'] == addr for e in node.listbanned()) + def run_test(self): # Node 0 connects to Node 1, check that the noban permission is not granted self.connect_nodes(0, 1) @@ -42,5 +45,18 @@ class SetBanTests(BitcoinTestFramework): peerinfo = self.nodes[1].getpeerinfo()[0] assert(not 'noban' in peerinfo['permissions']) + self.log.info("Test that a non-IP address can be banned/unbanned") + node = self.nodes[1] + tor_addr = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion" + ip_addr = "1.2.3.4" + assert(not self.is_banned(node, tor_addr)) + assert(not self.is_banned(node, ip_addr)) + node.setban(tor_addr, "add") + assert(self.is_banned(node, tor_addr)) + assert(not self.is_banned(node, ip_addr)) + node.setban(tor_addr, "remove") + assert(not self.is_banned(self.nodes[1], tor_addr)) + assert(not self.is_banned(node, ip_addr)) + if __name__ == '__main__': SetBanTests().main() diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessage.py index 0cb3ce4215..1c71732a61 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessage.py @@ -5,7 +5,10 @@ """Test RPC commands for signing and verifying messages.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) class SignMessagesTest(BitcoinTestFramework): def set_test_params(self): @@ -38,5 +41,22 @@ class SignMessagesTest(BitcoinTestFramework): assert not self.nodes[0].verifymessage(other_address, signature, message) assert not self.nodes[0].verifymessage(address, other_signature, message) + self.log.info('test parameter validity and error codes') + # signmessage(withprivkey) have two required parameters + for num_params in [0, 1, 3, 4, 5]: + param_list = ["dummy"]*num_params + assert_raises_rpc_error(-1, "signmessagewithprivkey", self.nodes[0].signmessagewithprivkey, *param_list) + assert_raises_rpc_error(-1, "signmessage", self.nodes[0].signmessage, *param_list) + # verifymessage has three required parameters + for num_params in [0, 1, 2, 4, 5]: + param_list = ["dummy"]*num_params + assert_raises_rpc_error(-1, "verifymessage", self.nodes[0].verifymessage, *param_list) + # invalid key or address provided + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[0].signmessagewithprivkey, "invalid_key", message) + assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].signmessage, "invalid_addr", message) + assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].verifymessage, "invalid_addr", signature, message) + # malformed signature provided + assert_raises_rpc_error(-3, "Malformed base64 encoding", self.nodes[0].verifymessage, self.nodes[0].getnewaddress(), "invalid_sig", message) + if __name__ == '__main__': SignMessagesTest().main() diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index e86f91b1d0..6177970872 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -10,6 +10,7 @@ Test corresponds to code in rpc/server.cpp. import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error class UptimeTest(BitcoinTestFramework): @@ -18,8 +19,12 @@ class UptimeTest(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): + self._test_negative_time() self._test_uptime() + def _test_negative_time(self): + assert_raises_rpc_error(-8, "Mocktime can not be negative: -1.", self.nodes[0].setmocktime, -1) + def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index f6ea9ef682..b8e899d675 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -178,7 +178,7 @@ can be called after the TestShell is shut down. | `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | | `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | | `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | -| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `setup_clean_chain` | `False` | A 200-block-long chain is initialized from cache by default. Instead, `setup_clean_chain` initializes an empty blockchain if set to `True`. | | `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | | `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | | `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py index 9de358aa0a..97b9c1d6d0 100644 --- a/test/functional/test_framework/bdb.py +++ b/test/functional/test_framework/bdb.py @@ -51,7 +51,6 @@ def dump_leaf_page(data): page_info['pgno'] = pgno page_info['prev_pgno'] = prev_pgno page_info['next_pgno'] = next_pgno - page_info['entries'] = entries page_info['hf_offset'] = hf_offset page_info['level'] = level page_info['pg_type'] = pg_type diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index e0cbab45ce..26526e35fa 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -20,10 +20,6 @@ def TaggedHash(tag, data): ss += data return hashlib.sha256(ss).digest() -def xor_bytes(b0, b1): - assert len(b0) == len(b1) - return bytes(x ^ y for (x, y) in zip(b0, b1)) - def jacobi_symbol(n, k): """Compute the Jacobi symbol of n modulo k @@ -510,7 +506,7 @@ class TestFrameworkKey(unittest.TestCase): if pubkey is not None: keys[privkey] = pubkey for msg in byte_arrays: # test every combination of message, signing key, verification key - for sign_privkey, sign_pubkey in keys.items(): + for sign_privkey, _ in keys.items(): sig = sign_schnorr(sign_privkey, msg) for verify_privkey, verify_pubkey in keys.items(): if verify_privkey == sign_privkey: diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index bab4ad0008..a18a9ec109 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -31,11 +31,6 @@ import time from test_framework.siphash import siphash256 from test_framework.util import hex_str_to_bytes, assert_equal -MIN_VERSION_SUPPORTED = 60001 -MY_VERSION = 70016 # past wtxid relay -MY_SUBVERSION = b"/python-p2p-tester:0.0.3/" -MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) - MAX_LOCATOR_SZ = 101 MAX_BLOCK_BASE_SIZE = 1000000 MAX_BLOOM_FILTER_SIZE = 36000 @@ -326,22 +321,20 @@ class CBlockLocator: __slots__ = ("nVersion", "vHave") def __init__(self): - self.nVersion = MY_VERSION self.vHave = [] def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] + struct.unpack("<i", f.read(4))[0] # Ignore version field. self.vHave = deser_uint256_vector(f) def serialize(self): r = b"" - r += struct.pack("<i", self.nVersion) + r += struct.pack("<i", 0) # Bitcoin Core ignores version field. Set it to 0. r += ser_uint256_vector(self.vHave) return r def __repr__(self): - return "CBlockLocator(nVersion=%i vHave=%s)" \ - % (self.nVersion, repr(self.vHave)) + return "CBlockLocator(vHave=%s)" % (repr(self.vHave)) class COutPoint: @@ -564,6 +557,9 @@ class CTransaction: def serialize(self): return self.serialize_with_witness() + def getwtxid(self): + return hash256(self.serialize())[::-1].hex() + # Recalculate the txid (transaction hash without witness) def rehash(self): self.sha256 = None @@ -579,7 +575,7 @@ class CTransaction: if self.sha256 is None: self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) - self.hash = encode(hash256(self.serialize_without_witness())[::-1], 'hex_codec').decode('ascii') + self.hash = hash256(self.serialize_without_witness())[::-1].hex() def is_valid(self): self.calc_sha256() @@ -1020,20 +1016,20 @@ class CMerkleBlock: # Objects that correspond to messages on the wire class msg_version: - __slots__ = ("addrFrom", "addrTo", "nNonce", "nRelay", "nServices", + __slots__ = ("addrFrom", "addrTo", "nNonce", "relay", "nServices", "nStartingHeight", "nTime", "nVersion", "strSubVer") msgtype = b"version" def __init__(self): - self.nVersion = MY_VERSION - self.nServices = NODE_NETWORK | NODE_WITNESS + self.nVersion = 0 + self.nServices = 0 self.nTime = int(time.time()) self.addrTo = CAddress() self.addrFrom = CAddress() self.nNonce = random.getrandbits(64) - self.strSubVer = MY_SUBVERSION + self.strSubVer = '' self.nStartingHeight = -1 - self.nRelay = MY_RELAY + self.relay = 0 def deserialize(self, f): self.nVersion = struct.unpack("<i", f.read(4))[0] @@ -1045,18 +1041,18 @@ class msg_version: self.addrFrom = CAddress() self.addrFrom.deserialize(f, with_time=False) self.nNonce = struct.unpack("<Q", f.read(8))[0] - self.strSubVer = deser_string(f) + self.strSubVer = deser_string(f).decode('utf-8') self.nStartingHeight = struct.unpack("<i", f.read(4))[0] if self.nVersion >= 70001: # Relay field is optional for version 70001 onwards try: - self.nRelay = struct.unpack("<b", f.read(1))[0] + self.relay = struct.unpack("<b", f.read(1))[0] except: - self.nRelay = 0 + self.relay = 0 else: - self.nRelay = 0 + self.relay = 0 def serialize(self): r = b"" @@ -1066,16 +1062,16 @@ class msg_version: r += self.addrTo.serialize(with_time=False) r += self.addrFrom.serialize(with_time=False) r += struct.pack("<Q", self.nNonce) - r += ser_string(self.strSubVer) + r += ser_string(self.strSubVer.encode('utf-8')) r += struct.pack("<i", self.nStartingHeight) - r += struct.pack("<b", self.nRelay) + r += struct.pack("<b", self.relay) return r def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i relay=%i)' \ % (self.nVersion, self.nServices, time.ctime(self.nTime), repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight, self.nRelay) + self.strSubVer, self.nStartingHeight, self.relay) class msg_verack: @@ -1270,7 +1266,7 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire # note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("msgtype", "data") + __slots__ = ("data") def __init__(self, msgtype, data=None): self.msgtype = msgtype diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index ea769ddfa2..05099f3339 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -31,7 +31,6 @@ import threading from test_framework.messages import ( CBlockHeader, MAX_HEADERS_RESULTS, - MIN_VERSION_SUPPORTED, msg_addr, msg_addrv2, msg_block, @@ -71,10 +70,26 @@ from test_framework.messages import ( NODE_WITNESS, sha256, ) -from test_framework.util import wait_until_helper +from test_framework.util import ( + MAX_NODES, + p2p_port, + wait_until_helper, +) logger = logging.getLogger("TestFramework.p2p") +# The minimum P2P version that this test framework supports +MIN_P2P_VERSION_SUPPORTED = 60001 +# The P2P version that this test framework implements and sends in its `version` message +# Version 70016 supports wtxid relay +P2P_VERSION = 70016 +# The services that this test framework offers in its `version` message +P2P_SERVICES = NODE_NETWORK | NODE_WITNESS +# The P2P user agent string that this test framework sends in its `version` message +P2P_SUBVERSION = "/python-p2p-tester:0.0.3/" +# Value for relay that this test framework sends in its `version` message +P2P_VERSION_RELAY = 1 + MESSAGEMAP = { b"addr": msg_addr, b"addrv2": msg_addrv2, @@ -139,7 +154,7 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): + def peer_connect_helper(self, dstaddr, dstport, net, timeout_factor): assert not self.is_connected self.timeout_factor = timeout_factor self.dstaddr = dstaddr @@ -148,12 +163,20 @@ class P2PConnection(asyncio.Protocol): self.on_connection_send_msg = None self.recvbuf = b"" self.magic_bytes = MAGIC_BYTES[net] - logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) + + def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): + self.peer_connect_helper(dstaddr, dstport, net, timeout_factor) loop = NetworkThread.network_event_loop - conn_gen_unsafe = loop.create_connection(lambda: self, host=self.dstaddr, port=self.dstport) - conn_gen = lambda: loop.call_soon_threadsafe(loop.create_task, conn_gen_unsafe) - return conn_gen + logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) + coroutine = loop.create_connection(lambda: self, host=self.dstaddr, port=self.dstport) + return lambda: loop.call_soon_threadsafe(loop.create_task, coroutine) + + def peer_accept_connection(self, connect_id, connect_cb=lambda: None, *, net, timeout_factor): + self.peer_connect_helper('0', 0, net, timeout_factor) + + logger.debug('Listening for Bitcoin Node with id: {}'.format(connect_id)) + return lambda: NetworkThread.listen(self, connect_cb, idx=connect_id) def peer_disconnect(self): # Connection could have already been closed by other end. @@ -312,18 +335,30 @@ class P2PInterface(P2PConnection): # If the peer supports wtxid-relay self.wtxidrelay = wtxidrelay - def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs): + def peer_connect_send_version(self, services): + # Send a version msg + vt = msg_version() + vt.nVersion = P2P_VERSION + vt.strSubVer = P2P_SUBVERSION + vt.relay = P2P_VERSION_RELAY + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.on_connection_send_msg = vt # Will be sent in connection_made callback + + def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) if send_version: - # Send a version msg - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.on_connection_send_msg = vt # Will be sent soon after connection_made + self.peer_connect_send_version(services) + + return create_conn + + def peer_accept_connection(self, *args, services=NODE_NETWORK | NODE_WITNESS, **kwargs): + create_conn = super().peer_accept_connection(*args, **kwargs) + self.peer_connect_send_version(services) return create_conn @@ -396,7 +431,7 @@ class P2PInterface(P2PConnection): pass def on_version(self, message): - assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) + assert message.nVersion >= MIN_P2P_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_P2P_VERSION_SUPPORTED) if message.nVersion >= 70016 and self.wtxidrelay: self.send_message(msg_wtxidrelay()) if self.support_addrv2: @@ -414,6 +449,10 @@ class P2PInterface(P2PConnection): wait_until_helper(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) + def wait_for_connect(self, timeout=60): + test_function = lambda: self.is_connected + wait_until_helper(test_function, timeout=timeout, lock=p2p_lock) + def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) @@ -527,6 +566,8 @@ class NetworkThread(threading.Thread): # There is only one event loop and no more than one thread must be created assert not self.network_event_loop + NetworkThread.listeners = {} + NetworkThread.protos = {} NetworkThread.network_event_loop = asyncio.new_event_loop() def run(self): @@ -542,6 +583,48 @@ class NetworkThread(threading.Thread): # Safe to remove event loop. NetworkThread.network_event_loop = None + @classmethod + def listen(cls, p2p, callback, port=None, addr=None, idx=1): + """ Ensure a listening server is running on the given port, and run the + protocol specified by `p2p` on the next connection to it. Once ready + for connections, call `callback`.""" + + if port is None: + assert 0 < idx <= MAX_NODES + port = p2p_port(MAX_NODES - idx) + if addr is None: + addr = '127.0.0.1' + + coroutine = cls.create_listen_server(addr, port, callback, p2p) + cls.network_event_loop.call_soon_threadsafe(cls.network_event_loop.create_task, coroutine) + + @classmethod + async def create_listen_server(cls, addr, port, callback, proto): + def peer_protocol(): + """Returns a function that does the protocol handling for a new + connection. To allow different connections to have different + behaviors, the protocol function is first put in the cls.protos + dict. When the connection is made, the function removes the + protocol function from that dict, and returns it so the event loop + can start executing it.""" + response = cls.protos.get((addr, port)) + cls.protos[(addr, port)] = None + return response + + if (addr, port) not in cls.listeners: + # When creating a listener on a given (addr, port) we only need to + # do it once. If we want different behaviors for different + # connections, we can accomplish this by providing different + # `proto` functions + + listener = await cls.network_event_loop.create_server(peer_protocol, addr, port) + logger.debug("Listening server on %s:%d should be started" % (addr, port)) + cls.listeners[(addr, port)] = listener + + cls.protos[(addr, port)] = proto + callback(addr, port) + + class P2PDataStore(P2PInterface): """A P2P data store class. diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index be0e9f24e2..c35533698c 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -29,8 +29,6 @@ MAX_SCRIPT_ELEMENT_SIZE = 520 LOCKTIME_THRESHOLD = 500000000 ANNEX_TAG = 0x50 -OPCODE_NAMES = {} # type: Dict[CScriptOp, str] - LEAF_VERSION_TAPSCRIPT = 0xc0 def hash160(s): @@ -47,7 +45,6 @@ def bn2vch(v): # Serialize to bytes return encoded_v.to_bytes(n_bytes, 'little') -_opcode_instances = [] # type: List[CScriptOp] class CScriptOp(int): """A single script opcode""" __slots__ = () @@ -111,6 +108,9 @@ class CScriptOp(int): _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] +OPCODE_NAMES: Dict[CScriptOp, str] = {} +_opcode_instances: List[CScriptOp] = [] + # Populate opcode instance table for n in range(0xff + 1): CScriptOp(n) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 7c5317480c..02eb10b5a4 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -18,6 +18,8 @@ import sys import tempfile import time +from typing import List +from .address import ADDRESS_BCRT1_P2WSH_OP_TRUE from .authproxy import JSONRPCException from . import coverage from .p2p import NetworkThread @@ -89,14 +91,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" - chain = None # type: str - setup_clean_chain = None # type: bool - def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" - self.chain = 'regtest' - self.setup_clean_chain = False - self.nodes = [] + self.chain: str = 'regtest' + self.setup_clean_chain: bool = False + self.nodes: List[TestNode] = [] self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = True @@ -110,6 +109,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # skipped. If list is truncated, wallet creation is skipped and keys # are not imported. self.wallet_names = None + # By default the wallet is not required. Set to true by skip_if_no_wallet(). + # When False, we ignore wallet_names regardless of what it is. + self.requires_wallet = False self.set_test_params() assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes if self.options.timeout_factor == 0 : @@ -186,15 +188,30 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') group = parser.add_mutually_exclusive_group() - group.add_argument("--descriptors", default=False, action="store_true", + group.add_argument("--descriptors", action='store_const', const=True, help="Run test using a descriptor wallet", dest='descriptors') - group.add_argument("--legacy-wallet", default=False, action="store_false", + group.add_argument("--legacy-wallet", action='store_const', const=False, help="Run test using legacy wallets", dest='descriptors') self.add_options(parser) self.options = parser.parse_args() self.options.previous_releases_path = previous_releases_path + config = configparser.ConfigParser() + config.read_file(open(self.options.configfile)) + self.config = config + + if self.options.descriptors is None: + # Prefer BDB unless it isn't available + if self.is_bdb_compiled(): + self.options.descriptors = False + elif self.is_sqlite_compiled(): + self.options.descriptors = True + else: + # If neither are compiled, tests requiring a wallet will be skipped and the value of self.options.descriptors won't matter + # It still needs to exist and be None in order for tests to work however. + self.options.descriptors = None + def setup(self): """Call this method to start up the test framework object with options set.""" @@ -204,9 +221,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.cachedir = os.path.abspath(self.options.cachedir) - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - self.config = config + config = self.config + fname_bitcoind = os.path.join( config["environment"]["BUILDDIR"], "src", @@ -379,7 +395,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() - if self.is_wallet_compiled(): + if self.requires_wallet: self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: @@ -717,16 +733,17 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Set a time in the past, so that blocks don't end up in the future cache_node.setmocktime(cache_node.getblockheader(cache_node.getbestblockhash())['time']) - # Create a 199-block-long chain; each of the 4 first nodes + # Create a 199-block-long chain; each of the 3 first nodes # gets 25 mature blocks and 25 immature. - # The 4th node gets only 24 immature blocks so that the very last + # The 4th address gets 25 mature and only 24 immature blocks so that the very last # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). + gen_addresses = [k.address for k in TestNode.PRIV_KEYS] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] for i in range(8): cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, - address=TestNode.PRIV_KEYS[i % 4].address, + address=gen_addresses[i % 4], ) assert_equal(cache_node.getblockchaininfo()["blocks"], 199) @@ -771,10 +788,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def skip_if_no_wallet(self): """Skip the running test if wallet has not been compiled.""" + self.requires_wallet = True if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") if self.options.descriptors: self.skip_if_no_sqlite() + else: + self.skip_if_no_bdb() def skip_if_no_sqlite(self): """Skip the running test if sqlite has not been compiled.""" @@ -809,10 +829,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.previous_releases_path)) return self.options.prev_releases + def skip_if_no_external_signer(self): + """Skip the running test if external signer support has not been compiled.""" + if not self.is_external_signer_compiled(): + raise SkipTest("external signer support has not been compiled.") + def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") + def is_external_signer_compiled(self): + """Checks whether external signer support was compiled.""" + return self.config["components"].getboolean("ENABLE_EXTERNAL_SIGNER") + def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e10ec1328b..ce9c1bc024 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -23,9 +23,10 @@ import sys from .authproxy import JSONRPCException from .descriptors import descsum_create -from .messages import MY_SUBVERSION +from .p2p import P2P_SUBVERSION from .util import ( MAX_NODES, + assert_equal, append_config, delete_cookie_file, get_auth_cookie, @@ -71,6 +72,7 @@ class TestNode(): """ self.index = i + self.p2p_conn_index = 1 self.datadir = datadir self.bitcoinconf = os.path.join(self.datadir, "bitcoin.conf") self.stdout_dir = os.path.join(self.datadir, "stdout") @@ -113,6 +115,8 @@ class TestNode(): if self.version_is_at_least(190000): self.args.append("-logthreadnames") + if self.version_is_at_least(219900): + self.args.append("-logsourcelocations") self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli @@ -517,7 +521,7 @@ class TestNode(): self._raise_assertion_error(assert_msg) def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): - """Add a p2p connection to the node. + """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also returns the connection to the caller.""" @@ -544,17 +548,46 @@ class TestNode(): # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely. p2p_conn.sync_with_ping() + # Consistency check that the Bitcoin Core has received our user agent string. This checks the + # node's newest peer. It could be racy if another Bitcoin Core node has connected since we opened + # our connection, but we don't expect that to happen. + assert_equal(self.getpeerinfo()[-1]['subver'], P2P_SUBVERSION) + + return p2p_conn + + def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): + """Add an outbound p2p connection from node. Either + full-relay("outbound-full-relay") or + block-relay-only("block-relay-only") connection. + + This method adds the p2p connection to the self.p2ps list and returns + the connection to the caller. + """ + + def addconnection_callback(address, port): + self.log.debug("Connecting to %s:%d %s" % (address, port, connection_type)) + self.addconnection('%s:%d' % (address, port), connection_type) + + p2p_conn.peer_accept_connection(connect_cb=addconnection_callback, connect_id=p2p_idx + 1, net=self.chain, timeout_factor=self.timeout_factor, **kwargs)() + + p2p_conn.wait_for_connect() + self.p2ps.append(p2p_conn) + + p2p_conn.wait_for_verack() + p2p_conn.sync_with_ping() + return p2p_conn def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" - return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION.decode("utf-8")]) + return len([peer for peer in self.getpeerinfo() if peer['subver'] == P2P_SUBVERSION]) def disconnect_p2ps(self): """Close all p2p connections to the node.""" for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] + wait_until_helper(lambda: self.num_test_p2p_connections() == 0, timeout_factor=self.timeout_factor) @@ -645,10 +678,10 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None): + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None, external_signer=None): if descriptors is None: descriptors = self.descriptors - return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup) + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup, external_signer) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index b3eb2d61a7..d335d4ea79 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -342,16 +342,25 @@ def initialize_datadir(dirname, n, chain): datadir = get_datadir_path(dirname, n) if not os.path.isdir(datadir): os.makedirs(datadir) - # Translate chain name to config name + write_config(os.path.join(datadir, "bitcoin.conf"), n=n, chain=chain) + os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) + os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) + return datadir + + +def write_config(config_path, *, n, chain, extra_config=""): + # Translate chain subdirectory name to config name if chain == 'testnet3': chain_name_conf_arg = 'testnet' chain_name_conf_section = 'test' else: chain_name_conf_arg = chain chain_name_conf_section = chain - with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: - f.write("{}=1\n".format(chain_name_conf_arg)) - f.write("[{}]\n".format(chain_name_conf_section)) + with open(config_path, 'w', encoding='utf8') as f: + if chain_name_conf_arg: + f.write("{}=1\n".format(chain_name_conf_arg)) + if chain_name_conf_section: + f.write("[{}]\n".format(chain_name_conf_section)) f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") f.write("fallbackfee=0.0002\n") @@ -359,14 +368,13 @@ def initialize_datadir(dirname, n, chain): f.write("keypool=1\n") f.write("discover=0\n") f.write("dnsseed=0\n") + f.write("fixedseeds=0\n") f.write("listenonion=0\n") f.write("printtoconsole=0\n") f.write("upnp=0\n") f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") - os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) - os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) - return datadir + f.write(extra_config) def get_datadir_path(dirname, n): diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index a71f2c69cb..38fbf3c1a6 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -32,6 +32,15 @@ class MiniWallet: self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE self._scriptPubKey = hex_str_to_bytes(self._test_node.validateaddress(self._address)['scriptPubKey']) + def scan_blocks(self, *, start=1, num): + """Scan the blocks for self._address outputs and add them to self._utxos""" + for i in range(start, start + num): + block = self._test_node.getblock(blockhash=self._test_node.getblockhash(i), verbosity=2) + for tx in block['tx']: + for out in tx['vout']: + if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): + self._utxos.append({'txid': tx['txid'], 'vout': out['n'], 'value': out['value']}) + def generate(self, num_blocks): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" blocks = self._test_node.generatetoaddress(num_blocks, self._address) @@ -71,9 +80,9 @@ class MiniWallet: tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] tx_hex = tx.serialize().hex() - txid = from_node.sendrawtransaction(tx_hex) - self._utxos.append({'txid': txid, 'vout': 0, 'value': send_value}) - tx_info = from_node.getmempoolentry(txid) + tx_info = from_node.testmempoolaccept([tx_hex])[0] + self._utxos.append({'txid': tx_info['txid'], 'vout': 0, 'value': send_value}) + from_node.sendrawtransaction(tx_hex) assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fee'], fee) - return {'txid': txid, 'wtxid': tx_info['wtxid'], 'hex': tx_hex} + assert_equal(tx_info['fees']['base'], fee) + return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex} diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 261c1f0a1b..79ad2cf161 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -86,58 +86,60 @@ EXTENDED_SCRIPTS = [ BASE_SCRIPTS = [ # Scripts that are run by default. # Longest test should go first, to favor running tests in parallel - 'wallet_hd.py', + 'wallet_hd.py --legacy-wallet', 'wallet_hd.py --descriptors', - 'wallet_backup.py', + 'wallet_backup.py --legacy-wallet', 'wallet_backup.py --descriptors', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', 'feature_maxuploadtarget.py', 'feature_block.py', - 'rpc_fundrawtransaction.py', + 'rpc_fundrawtransaction.py --legacy-wallet', 'rpc_fundrawtransaction.py --descriptors', 'p2p_compactblocks.py', 'feature_segwit.py --legacy-wallet', # vv Tests less than 2m vv - 'wallet_basic.py', + 'wallet_basic.py --legacy-wallet', 'wallet_basic.py --descriptors', - 'wallet_labels.py', + 'wallet_labels.py --legacy-wallet', 'wallet_labels.py --descriptors', 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'wallet_listtransactions.py', + 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', 'feature_taproot.py', + 'wallet_signer.py --descriptors', # vv Tests less than 60s vv 'p2p_sendheaders.py', 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', - 'wallet_listreceivedby.py', + 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', - 'wallet_abandonconflict.py', + 'wallet_abandonconflict.py --legacy-wallet', 'wallet_abandonconflict.py --descriptors', 'feature_csv_activation.py', - 'rpc_rawtransaction.py', + 'rpc_rawtransaction.py --legacy-wallet', 'rpc_rawtransaction.py --descriptors', - 'wallet_address_types.py', + 'wallet_address_types.py --legacy-wallet', 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', 'p2p_feefilter.py', 'feature_reindex.py', 'feature_abortnode.py', # vv Tests less than 30s vv - 'wallet_keypool_topup.py', + 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', + 'rpc_invalid_address_message.py', 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', - 'tool_wallet.py', + 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', @@ -145,14 +147,14 @@ BASE_SCRIPTS = [ 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', - 'wallet_avoidreuse.py', + 'wallet_avoidreuse.py --legacy-wallet', 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', - 'wallet_multiwallet.py', + 'wallet_multiwallet.py --legacy-wallet', 'wallet_multiwallet.py --descriptors', 'wallet_multiwallet.py --usecli', - 'wallet_createwallet.py', + 'wallet_createwallet.py --legacy-wallet', 'wallet_createwallet.py --usecli', 'wallet_createwallet.py --descriptors', 'wallet_watchonly.py --legacy-wallet', @@ -160,27 +162,27 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', - 'rpc_psbt.py', + 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', - 'rpc_signrawtransaction.py', + 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', - 'wallet_groups.py', + 'wallet_groups.py --legacy-wallet', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', 'p2p_disconnect_ban.py', 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py', + 'wallet_disable.py --legacy-wallet', 'wallet_disable.py --descriptors', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', 'rpc_net.py', - 'wallet_keypool.py', + 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', 'p2p_nobloomfilter_messages.py', @@ -194,75 +196,80 @@ BASE_SCRIPTS = [ 'p2p_invalid_tx.py', 'feature_assumevalid.py', 'example_test.py', - 'wallet_txn_doublespend.py', + 'wallet_txn_doublespend.py --legacy-wallet', 'wallet_txn_doublespend.py --descriptors', - 'feature_backwards_compatibility.py', + 'feature_backwards_compatibility.py --legacy-wallet', 'feature_backwards_compatibility.py --descriptors', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', + 'feature_utxo_set_hash.py', 'feature_rbf.py', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py', + 'rpc_createmultisig.py --legacy-wallet', 'rpc_createmultisig.py --descriptors', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', - 'wallet_importprunedfunds.py', + 'wallet_importprunedfunds.py --legacy-wallet', 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', 'rpc_signmessage.py', 'rpc_generateblock.py', 'rpc_generate.py', - 'wallet_balance.py', + 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', - 'feature_nulldummy.py', + 'feature_nulldummy.py --legacy-wallet', 'feature_nulldummy.py --descriptors', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py --legacy-wallet', 'wallet_import_with_label.py --legacy-wallet', 'wallet_importdescriptors.py --descriptors', - 'wallet_upgradewallet.py', + 'wallet_upgradewallet.py --legacy-wallet', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', 'mining_basic.py', 'feature_signet.py', - 'wallet_bumpfee.py', + 'wallet_bumpfee.py --legacy-wallet', 'wallet_bumpfee.py --descriptors', 'wallet_implicitsegwit.py --legacy-wallet', 'rpc_named_arguments.py', - 'wallet_listsinceblock.py', + 'wallet_listsinceblock.py --legacy-wallet', 'wallet_listsinceblock.py --descriptors', + 'wallet_listdescriptors.py --descriptors', 'p2p_leak.py', - 'wallet_encryption.py', + 'wallet_encryption.py --legacy-wallet', 'wallet_encryption.py --descriptors', 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', - 'wallet_resendwallettransactions.py', + 'wallet_resendwallettransactions.py --legacy-wallet', 'wallet_resendwallettransactions.py --descriptors', - 'wallet_fallbackfee.py', + 'wallet_fallbackfee.py --legacy-wallet', 'wallet_fallbackfee.py --descriptors', 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', - 'wallet_create_tx.py', - 'wallet_send.py', + 'wallet_create_tx.py --legacy-wallet', + 'wallet_send.py --legacy-wallet', + 'wallet_send.py --descriptors', 'wallet_create_tx.py --descriptors', 'p2p_fingerprint.py', 'feature_uacomment.py', - 'wallet_coinbase_category.py', + 'wallet_coinbase_category.py --legacy-wallet', 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', 'p2p_dos_header_tree.py', + 'p2p_add_connections.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', + 'p2p_message_capture.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', @@ -283,6 +290,7 @@ BASE_SCRIPTS = [ 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', + 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 8a1af24dcf..28103793df 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -183,11 +183,13 @@ class ToolWalletTest(BitcoinTestFramework): def test_invalid_tool_commands_and_args(self): self.log.info('Testing that various invalid commands raise with specific error messages') - self.assert_raises_tool_error('Invalid command: foo', 'foo') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'foo'", 'foo') # `bitcoin-wallet help` raises an error. Use `bitcoin-wallet -help`. - self.assert_raises_tool_error('Invalid command: help', 'help') - self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'help'", 'help') + self.assert_raises_tool_error('Error: Additional arguments provided (create). Methods do not take arguments. Please refer to `-help`.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('No method provided. Run `bitcoin-wallet -help` for valid methods.') + self.assert_raises_tool_error('Wallet name must be provided when creating a new wallet.', 'create') locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: @@ -348,7 +350,8 @@ class ToolWalletTest(BitcoinTestFramework): self.log.info('Checking createfromdump') self.do_tool_createfromdump("load", "wallet.dump") - self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") + if self.is_bdb_compiled(): + self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") if self.is_sqlite_compiled(): self.do_tool_createfromdump("load-sqlite", "wallet.dump", "sqlite") diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 229c134a4b..bc4fa90e83 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -65,7 +65,6 @@ def assert_balances(node, mine): class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 6bcb03e8ed..4a589f0393 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -586,7 +586,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) # Test getaddressinfo on external address. Note that these addresses are taken from disablewallet.py - assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") address_info = self.nodes[0].getaddressinfo("mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info['address'], "mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info["scriptPubKey"], "76a9144e3854046c7bd1594ac904e4793b6a45b36dea0988ac") diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c8c1f2e374..5fc8438e8f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -425,6 +425,9 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): original_txid = watcher.sendrawtransaction(psbt_final["hex"]) assert_equal(len(watcher.decodepsbt(psbt)["tx"]["vin"]), 1) + # bumpfee can't be used on watchonly wallets + assert_raises_rpc_error(-4, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead.", watcher.bumpfee, original_txid) + # Bump fee, obnoxiously high to add additional watchonly input bumped_psbt = watcher.psbtbumpfee(original_txid, {"fee_rate": HIGH}) assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index cf3317121f..16a0a50b07 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -17,7 +17,6 @@ from test_framework.wallet_util import bytes_to_wif, generate_wif_key class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 1de41a5f96..1e032bdd6c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -23,11 +23,14 @@ class WalletDescriptorTest(BitcoinTestFramework): self.skip_if_no_sqlite() def run_test(self): - # Make a legacy wallet and check it is BDB - self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) - wallet_info = self.nodes[0].getwalletinfo() - assert_equal(wallet_info['format'], 'bdb') - self.nodes[0].unloadwallet("legacy1") + if self.is_bdb_compiled(): + # Make a legacy wallet and check it is BDB + self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) + wallet_info = self.nodes[0].getwalletinfo() + assert_equal(wallet_info['format'], 'bdb') + self.nodes[0].unloadwallet("legacy1") + else: + self.log.warning("Skipping BDB test") # Make a descriptor wallet self.log.info("Making a descriptor wallet") @@ -148,5 +151,62 @@ class WalletDescriptorTest(BitcoinTestFramework): nopriv_rpc = self.nodes[0].get_wallet_rpc('desc_no_priv') assert_raises_rpc_error(-4, 'This wallet has no available keys', nopriv_rpc.getnewaddress) + self.log.info("Test descriptor exports") + self.nodes[0].createwallet(wallet_name='desc_export', descriptors=True) + exp_rpc = self.nodes[0].get_wallet_rpc('desc_export') + self.nodes[0].createwallet(wallet_name='desc_import', disable_private_keys=True, descriptors=True) + imp_rpc = self.nodes[0].get_wallet_rpc('desc_import') + + addr_types = [('legacy', False, 'pkh(', '44\'/1\'/0\'', -13), + ('p2sh-segwit', False, 'sh(wpkh(', '49\'/1\'/0\'', -14), + ('bech32', False, 'wpkh(', '84\'/1\'/0\'', -13), + ('legacy', True, 'pkh(', '44\'/1\'/0\'', -13), + ('p2sh-segwit', True, 'sh(wpkh(', '49\'/1\'/0\'', -14), + ('bech32', True, 'wpkh(', '84\'/1\'/0\'', -13)] + + for addr_type, internal, desc_prefix, deriv_path, int_idx in addr_types: + int_str = 'internal' if internal else 'external' + + self.log.info("Testing descriptor address type for {} {}".format(addr_type, int_str)) + if internal: + addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + else: + addr = exp_rpc.getnewaddress(address_type=addr_type) + desc = exp_rpc.getaddressinfo(addr)['parent_desc'] + assert_equal(desc_prefix, desc[0:len(desc_prefix)]) + idx = desc.index('/') + 1 + assert_equal(deriv_path, desc[idx:idx + 9]) + if internal: + assert_equal('1', desc[int_idx]) + else: + assert_equal('0', desc[int_idx]) + + self.log.info("Testing the same descriptor is returned for address type {} {}".format(addr_type, int_str)) + for i in range(0, 10): + if internal: + addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + else: + addr = exp_rpc.getnewaddress(address_type=addr_type) + test_desc = exp_rpc.getaddressinfo(addr)['parent_desc'] + assert_equal(desc, test_desc) + + self.log.info("Testing import of exported {} descriptor".format(addr_type)) + imp_rpc.importdescriptors([{ + 'desc': desc, + 'active': True, + 'next_index': 11, + 'timestamp': 'now', + 'internal': internal + }]) + + for i in range(0, 10): + if internal: + exp_addr = exp_rpc.getrawchangeaddress(address_type=addr_type) + imp_addr = imp_rpc.getrawchangeaddress(address_type=addr_type) + else: + exp_addr = exp_rpc.getnewaddress(address_type=addr_type) + imp_addr = imp_rpc.getnewaddress(address_type=addr_type) + assert_equal(exp_addr, imp_addr) + if __name__ == '__main__': WalletDescriptorTest().main () diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py new file mode 100755 index 0000000000..8d02949ff4 --- /dev/null +++ b/test/functional/wallet_listdescriptors.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the listdescriptors RPC.""" + +from test_framework.descriptors import ( + descsum_create +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class ListDescriptorsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + # do not create any wallet by default + def init_wallet(self, i): + return + + def run_test(self): + node = self.nodes[0] + assert_raises_rpc_error(-18, 'No wallet is loaded.', node.listdescriptors) + + self.log.info('Test that the command is not available for legacy wallets.') + node.createwallet(wallet_name='w1', descriptors=False) + assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) + + self.log.info('Test the command for empty descriptors wallet.') + node.createwallet(wallet_name='w2', blank=True, descriptors=True) + assert_equal(0, len(node.get_wallet_rpc('w2').listdescriptors())) + + self.log.info('Test the command for a default descriptors wallet.') + node.createwallet(wallet_name='w3', descriptors=True) + result = node.get_wallet_rpc('w3').listdescriptors() + assert_equal(6, len(result)) + assert_equal(6, len([d for d in result if d['active']])) + assert_equal(3, len([d for d in result if d['internal']])) + for item in result: + assert item['desc'] != '' + assert item['next'] == 0 + assert item['range'] == [0, 0] + assert item['timestamp'] is not None + + self.log.info('Test descriptors with hardened derivations are listed in importable form.') + xprv = 'tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg' + xpub_acc = 'tpubDCMVLhErorrAGfApiJSJzEKwqeaf2z3NrkVMxgYQjZLzMjXMBeRw2muGNYbvaekAE8rUFLftyEar4LdrG2wXyyTJQZ26zptmeTEjPTaATts' + hardened_path = '/84\'/1\'/0\'' + wallet = node.get_wallet_rpc('w2') + wallet.importdescriptors([{ + 'desc': descsum_create('wpkh(' + xprv + hardened_path + '/0/*)'), + 'timestamp': 1296688602, + }]) + expected = {'desc': descsum_create('wpkh([80002067' + hardened_path + ']' + xpub_acc + '/0/*)'), + 'timestamp': 1296688602, + 'active': False, + 'range': [0, 0], + 'next': 0} + assert_equal([expected], wallet.listdescriptors()) + + self.log.info('Test non-active non-range combo descriptor') + node.createwallet(wallet_name='w4', blank=True, descriptors=True) + wallet = node.get_wallet_rpc('w4') + wallet.importdescriptors([{ + 'desc': descsum_create('combo(' + node.get_deterministic_priv_key().key + ')'), + 'timestamp': 1296688602, + }]) + expected = [{'active': False, + 'desc': 'combo(0227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3f)#np574htj', + 'timestamp': 1296688602}] + assert_equal(expected, wallet.listdescriptors()) + + +if __name__ == '__main__': + ListDescriptorsTest().main() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index bb89e76a9a..bf24b9c7b3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -304,12 +304,12 @@ class MultiWalletTest(BitcoinTestFramework): if self.options.descriptors: assert_raises_rpc_error(-4, "Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?", self.nodes[0].loadwallet, wallet_names[0]) else: - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) # This tests the default wallet that BDB makes, so SQLite wallet doesn't need to test this # Fail to load duplicate wallets by different ways (directory and filepath) path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary # Fail to load if one wallet is a copy of another diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 9835c5a2af..880341fdd9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -8,6 +8,7 @@ from decimal import Decimal, getcontext from itertools import product from test_framework.authproxy import JSONRPCException +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -168,49 +169,91 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name="w1") w1 = self.nodes[1].get_wallet_rpc("w1") # w2 contains the private keys for w3 - self.nodes[1].createwallet(wallet_name="w2") + self.nodes[1].createwallet(wallet_name="w2", blank=True) w2 = self.nodes[1].get_wallet_rpc("w2") + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + xpub = "tpubD6NzVbkrYhZ4YkEfMbRJkQyZe7wTkbTNRECozCtJPtdLRn6cT1QKb8yHjwAPcAr26eHBFYs5iLiFFnCbwPRsncCKUKCfubHDMGKzMVcN1Jg" + if self.options.descriptors: + w2.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpriv + "/0/0/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True + },{ + "desc": descsum_create("wpkh(" + xpriv + "/0/1/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True, + "internal": True + }]) + else: + w2.sethdseed(True) + # w3 is a watch-only wallet, based on w2 self.nodes[1].createwallet(wallet_name="w3", disable_private_keys=True) w3 = self.nodes[1].get_wallet_rpc("w3") - for _ in range(3): - a2_receive = w2.getnewaddress() - a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation - res = w3.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], + if self.options.descriptors: + # Match the privkeys in w2 for descriptors + res = w3.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpub + "/0/0/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "watchonly": True },{ - "desc": w2.getaddressinfo(a2_change)["desc"], + "desc": descsum_create("wpkh(" + xpub + "/0/1/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "internal": True, "watchonly": True }]) assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w3 - self.nodes[0].generate(1) - self.sync_blocks() - - # w4 has private keys enabled, but only contains watch-only keys (from w2) - self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) - w4 = self.nodes[1].get_wallet_rpc("w4") for _ in range(3): a2_receive = w2.getnewaddress() - res = w4.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], - "timestamp": "now", - "keypool": False, - "watchonly": True - }]) - assert_equal(res, [{"success": True}]) + if not self.options.descriptors: + # Because legacy wallets use exclusively hardened derivation, we can't do a ranged import like we do for descriptors + a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation + res = w3.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": True, + "watchonly": True + },{ + "desc": w2.getaddressinfo(a2_change)["desc"], + "timestamp": "now", + "keypool": True, + "internal": True, + "watchonly": True + }]) + assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w4 + w0.sendtoaddress(a2_receive, 10) # fund w3 self.nodes[0].generate(1) self.sync_blocks() + if not self.options.descriptors: + # w4 has private keys enabled, but only contains watch-only keys (from w2) + # This is legacy wallet behavior only as descriptor wallets don't allow watchonly and non-watchonly things in the same wallet. + self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) + w4 = self.nodes[1].get_wallet_rpc("w4") + for _ in range(3): + a2_receive = w2.getnewaddress() + res = w4.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": False, + "watchonly": True + }]) + assert_equal(res, [{"success": True}]) + + w0.sendtoaddress(a2_receive, 10) # fund w4 + self.nodes[0].generate(1) + self.sync_blocks() + self.log.info("Send to address...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=True) @@ -241,11 +284,15 @@ class WalletSendTest(BitcoinTestFramework): res = w2.walletprocesspsbt(res["psbt"]) assert res["complete"] - self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") - self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) - res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) - res = w2.walletprocesspsbt(res["psbt"]) - assert res["complete"] + if not self.options.descriptors: + # Descriptor wallets do not allow mixed watch-only and non-watch-only things in the same wallet. + # This is specifically testing that w4 ignores its own private keys and creates a psbt with send + # which is not something that needs to be tested in descriptor wallets. + self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") + self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) + res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] self.log.info("Create OP_RETURN...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py new file mode 100755 index 0000000000..9dd080dca9 --- /dev/null +++ b/test/functional/wallet_signer.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test external signer. + +Verify that a bitcoind node can use an external signer command +""" +import os +import platform + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class SignerTest(BitcoinTestFramework): + def mock_signer_path(self): + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mocks', 'signer.py') + if platform.system() == "Windows": + return "py " + path + else: + return path + + def set_test_params(self): + self.num_nodes = 4 + + self.extra_args = [ + [], + [f"-signer={self.mock_signer_path()}", '-keypool=10'], + [f"-signer={self.mock_signer_path()}", '-keypool=10'], + ["-signer=fake.py"], + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_external_signer() + + def set_mock_result(self, node, res): + with open(os.path.join(node.cwd, "mock_result"), "w", encoding="utf8") as f: + f.write(res) + + def clear_mock_result(self, node): + os.remove(os.path.join(node.cwd, "mock_result")) + + def run_test(self): + self.log.debug(f"-signer={self.mock_signer_path()}") + + assert_raises_rpc_error(-4, 'Error: restart bitcoind with -signer=<cmd>', + self.nodes[0].enumeratesigners + ) + + # Handle script missing: + assert_raises_rpc_error(-1, 'execve failed: No such file or directory', + self.nodes[3].enumeratesigners + ) + + # Handle error thrown by script + self.set_mock_result(self.nodes[1], "2") + assert_raises_rpc_error(-1, 'RunCommandParseJSON error', + self.nodes[1].enumeratesigners + ) + self.clear_mock_result(self.nodes[1]) + + self.set_mock_result(self.nodes[1], '0 [{"type": "trezor", "model": "trezor_t", "error": "fingerprint not found"}]') + assert_raises_rpc_error(-4, 'fingerprint not found', + self.nodes[1].enumeratesigners + ) + self.clear_mock_result(self.nodes[1]) + + # Create new wallets for an external signer. + # disable_private_keys and descriptors must be true: + assert_raises_rpc_error(-4, "Private keys must be disabled when using an external signer", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=False, descriptors=True, external_signer=True) + if self.is_bdb_compiled(): + assert_raises_rpc_error(-4, "Descriptor support must be enabled when using an external signer", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=True, descriptors=False, external_signer=True) + else: + assert_raises_rpc_error(-4, "Compiled without bdb support (required for legacy wallets)", self.nodes[1].createwallet, wallet_name='not_hww', disable_private_keys=True, descriptors=False, external_signer=True) + + self.nodes[1].createwallet(wallet_name='hww', disable_private_keys=True, descriptors=True, external_signer=True) + hww = self.nodes[1].get_wallet_rpc('hww') + + result = hww.enumeratesigners() + assert_equal(len(result['signers']), 2) + assert_equal(result['signers'][0]["fingerprint"], "00000001") + assert_equal(result['signers'][0]["name"], "trezor_t") + + # Flag can't be set afterwards (could be added later for non-blank descriptor based watch-only wallets) + self.nodes[1].createwallet(wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=False) + not_hww = self.nodes[1].get_wallet_rpc('not_hww') + assert_raises_rpc_error(-8, "Wallet flag is immutable: external_signer", not_hww.setwalletflag, "external_signer", True) + + # assert_raises_rpc_error(-4, "Multiple signers found, please specify which to use", wallet_name='not_hww', disable_private_keys=True, descriptors=True, external_signer=True) + + # TODO: Handle error thrown by script + # self.set_mock_result(self.nodes[1], "2") + # assert_raises_rpc_error(-1, 'Unable to parse JSON', + # self.nodes[1].createwallet, wallet_name='not_hww2', disable_private_keys=True, descriptors=True, external_signer=False + # ) + # self.clear_mock_result(self.nodes[1]) + + assert_equal(hww.getwalletinfo()["keypoolsize"], 30) + + address1 = hww.getnewaddress(address_type="bech32") + assert_equal(address1, "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g") + address_info = hww.getaddressinfo(address1) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/84'/1'/0'/0/0") + + address2 = hww.getnewaddress(address_type="p2sh-segwit") + assert_equal(address2, "2N2gQKzjUe47gM8p1JZxaAkTcoHPXV6YyVp") + address_info = hww.getaddressinfo(address2) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/49'/1'/0'/0/0") + + address3 = hww.getnewaddress(address_type="legacy") + assert_equal(address3, "n1LKejAadN6hg2FrBXoU1KrwX4uK16mco9") + address_info = hww.getaddressinfo(address3) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + + self.log.info('Test signerdisplayaddress') + result = hww.signerdisplayaddress(address1) + assert_equal(result, {"address": address1}) + + # Handle error thrown by script + self.set_mock_result(self.nodes[1], "2") + assert_raises_rpc_error(-1, 'RunCommandParseJSON error', + hww.signerdisplayaddress, address1 + ) + self.clear_mock_result(self.nodes[1]) + + self.log.info('Prepare mock PSBT') + self.nodes[0].sendtoaddress(address1, 1) + self.nodes[0].generate(1) + self.sync_all() + + # Load private key into wallet to generate a signed PSBT for the mock + self.nodes[1].createwallet(wallet_name="mock", disable_private_keys=False, blank=True, descriptors=True) + mock_wallet = self.nodes[1].get_wallet_rpc("mock") + assert mock_wallet.getwalletinfo()['private_keys_enabled'] + + result = mock_wallet.importdescriptors([{ + "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0", + "timestamp": 0, + "range": [0,1], + "internal": False, + "active": True + }, + { + "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh", + "timestamp": 0, + "range": [0, 0], + "internal": True, + "active": True + }]) + assert_equal(result[0], {'success': True}) + assert_equal(result[1], {'success': True}) + assert_equal(mock_wallet.getwalletinfo()["txcount"], 1) + dest = self.nodes[0].getnewaddress(address_type='bech32') + mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {}, True)['psbt'] + mock_psbt_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt, sign=True, sighashtype="ALL", bip32derivs=True) + mock_psbt_final = mock_wallet.finalizepsbt(mock_psbt_signed["psbt"]) + mock_tx = mock_psbt_final["hex"] + assert(mock_wallet.testmempoolaccept([mock_tx])[0]["allowed"]) + + # # Create a new wallet and populate with specific public keys, in order + # # to work with the mock signed PSBT. + # self.nodes[1].createwallet(wallet_name="hww4", disable_private_keys=True, descriptors=True, external_signer=True) + # hww4 = self.nodes[1].get_wallet_rpc("hww4") + # + # descriptors = [{ + # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/0/*)#x30uthjs", + # "timestamp": "now", + # "range": [0, 1], + # "internal": False, + # "watchonly": True, + # "active": True + # }, + # { + # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)#h92akzzg", + # "timestamp": "now", + # "range": [0, 0], + # "internal": True, + # "watchonly": True, + # "active": True + # }] + + # result = hww4.importdescriptors(descriptors) + # assert_equal(result[0], {'success': True}) + # assert_equal(result[1], {'success': True}) + assert_equal(hww.getwalletinfo()["txcount"], 1) + + assert(hww.testmempoolaccept([mock_tx])[0]["allowed"]) + + with open(os.path.join(self.nodes[1].cwd, "mock_psbt"), "w", encoding="utf8") as f: + f.write(mock_psbt_signed["psbt"]) + + self.log.info('Test send using hww1') + + res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) + assert(res["complete"]) + assert_equal(res["hex"], mock_tx) + + # # Handle error thrown by script + # self.set_mock_result(self.nodes[4], "2") + # assert_raises_rpc_error(-1, 'Unable to parse JSON', + # hww4.signerprocesspsbt, psbt_orig, "00000001" + # ) + # self.clear_mock_result(self.nodes[4]) + +if __name__ == '__main__': + SignerTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 893a2d9617..6fc1d13c53 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -11,9 +11,10 @@ from test_framework.util import ( ) from test_framework.messages import CTransaction, COIN + class TxnMallTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 4 + self.num_nodes = 3 self.supports_cli = False def skip_test_if_missing_module(self): @@ -38,9 +39,8 @@ class TxnMallTest(BitcoinTestFramework): # All nodes should start with 1,250 BTC: starting_balance = 1250 - for i in range(4): + for i in range(3): assert_equal(self.nodes[i].getbalance(), starting_balance) - self.nodes[i].getnewaddress() # bug workaround, coins generated assigned to first getnewaddress! self.nodes[0].settxfee(.001) @@ -139,5 +139,6 @@ class TxnMallTest(BitcoinTestFramework): expected -= 50 assert_equal(self.nodes[0].getbalance(), expected) + if __name__ == '__main__': TxnMallTest().main() diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index c7f7a8546a..0cb7328948 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -11,9 +11,10 @@ from test_framework.util import ( find_output, ) + class TxnMallTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 4 + self.num_nodes = 3 self.supports_cli = False def skip_test_if_missing_module(self): @@ -39,9 +40,8 @@ class TxnMallTest(BitcoinTestFramework): for n in self.nodes: assert n.getblockchaininfo()["initialblockdownload"] == False - for i in range(4): + for i in range(3): assert_equal(self.nodes[i].getbalance(), starting_balance) - self.nodes[i].getnewaddress("") # bug workaround, coins generated assigned to first getnewaddress! # Assign coins to foo and bar addresses: node0_address_foo = self.nodes[0].getnewaddress() @@ -136,5 +136,6 @@ class TxnMallTest(BitcoinTestFramework): # Node1's balance should be its initial balance (1250 for 25 block rewards) plus the doublespend: assert_equal(self.nodes[1].getbalance(), 1250 + 1240) + if __name__ == '__main__': TxnMallTest().main() diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index d0bb6135a8..fbc0f995d2 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -57,6 +57,7 @@ class UpgradeWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_bdb() self.skip_if_no_previous_releases() def setup_network(self): diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index b0c41b2738..c345c382d0 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -2,7 +2,7 @@ # Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test createwallet arguments. +"""Test createwallet watchonly arguments. """ from test_framework.test_framework import BitcoinTestFramework @@ -14,7 +14,6 @@ from test_framework.util import ( class CreateWalletWatchonlyTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): @@ -50,6 +49,11 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): assert_equal(len(wo_wallet.listtransactions()), 1) assert_equal(wo_wallet.getbalance(include_watchonly=False), 0) + self.log.info('Test sending from a watch-only wallet raises RPC error') + msg = "Error: Private keys are disabled for this wallet" + assert_raises_rpc_error(-4, msg, wo_wallet.sendtoaddress, a1, 0.1) + assert_raises_rpc_error(-4, msg, wo_wallet.sendmany, amounts={a1: 0.1}) + self.log.info('Testing listreceivedbyaddress watch-only defaults') result = wo_wallet.listreceivedbyaddress() assert_equal(len(result), 1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 3c743603bb..611061072f 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -14,6 +14,16 @@ import subprocess import sys +def get_fuzz_env(*, target, source_dir): + return { + 'FUZZ': target, + 'UBSAN_OPTIONS': + f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'ASAN_OPTIONS': # symbolizer disabled due to https://github.com/google/sanitizers/issues/1364#issuecomment-761072085 + 'symbolize=0:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1', + } + + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -129,9 +139,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), '-help=1', ], - env={ - 'FUZZ': test_list_selection[0] - }, + env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), timeout=20, check=True, stderr=subprocess.PIPE, @@ -148,6 +156,7 @@ def main(): if args.generate: return generate_corpus_seeds( fuzz_pool=fuzz_pool, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], seed_dir=args.seed_dir, targets=test_list_selection, @@ -158,6 +167,7 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], merge_dir=args.m_dir, ) @@ -167,12 +177,13 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], use_valgrind=args.valgrind, ) -def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): +def generate_corpus_seeds(*, fuzz_pool, src_dir, build_dir, seed_dir, targets): """Generates new corpus seeds. Run {targets} without input, and outputs the generated corpus seeds to @@ -186,9 +197,7 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): ' '.join(command), subprocess.run( command, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -209,13 +218,15 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): future.result() -def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir): logging.info("Merge the inputs from the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), '-merge=1', + '-shuffle=0', + '-prefer_small=1', '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), os.path.join(merge_dir, t), @@ -227,9 +238,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): output = 'Run {} with args {}\n'.format(t, " ".join(args)) output += subprocess.run( args, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -242,7 +251,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): future.result() -def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) @@ -257,7 +266,12 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): def job(t, args): output = 'Run {} with args {}'.format(t, args) - result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True) + result = subprocess.run( + args, + env=get_fuzz_env(target=t, source_dir=src_dir), + stderr=subprocess.PIPE, + universal_newlines=True, + ) output += result.stderr return output, result diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 1348b8246b..b177fbb4b2 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -175,6 +175,7 @@ def check_host(args) -> int: './depends/config.guess').decode()) if args.download_binary: platforms = { + 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', 'x86_64-apple-darwin*': 'osx64', } diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py deleted file mode 100755 index 0a4cc875d0..0000000000 --- a/test/lint/check-rpc-mappings.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Check RPC argument consistency.""" - -from collections import defaultdict -import os -import re -import sys - -# Source files (relative to root) to scan for dispatch tables -SOURCES = [ - "src/rpc/server.cpp", - "src/rpc/blockchain.cpp", - "src/rpc/mining.cpp", - "src/rpc/misc.cpp", - "src/rpc/net.cpp", - "src/rpc/rawtransaction.cpp", - "src/wallet/rpcwallet.cpp", -] -# Source file (relative to root) containing conversion mapping -SOURCE_CLIENT = 'src/rpc/client.cpp' -# Argument names that should be ignored in consistency checks -IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'} - -class RPCCommand: - def __init__(self, name, args): - self.name = name - self.args = args - -class RPCArgument: - def __init__(self, names, idx): - self.names = names - self.idx = idx - self.convert = False - -def parse_string(s): - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - -def process_commands(fname): - """Find and parse dispatch table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if re.match(r"static const CRPCCommand .*\[\] =", line): - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(2)) - args_str = m.group(4).strip() - if args_str: - args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))] - else: - args = [] - cmds.append(RPCCommand(name, args)) - assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps" - return cmds - -def process_mapping(fname): - """Find and parse conversion table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if line == 'static const CRPCConvertParam vRPCConvertParams[] =': - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(1)) - idx = int(m.group(2)) - argname = parse_string(m.group(3)) - cmds.append((name, idx, argname)) - assert not in_rpcs and cmds - return cmds - -def main(): - if len(sys.argv) != 2: - print('Usage: {} ROOT-DIR'.format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - root = sys.argv[1] - - # Get all commands from dispatch tables - cmds = [] - for fname in SOURCES: - cmds += process_commands(os.path.join(root, fname)) - - cmds_by_name = {} - for cmd in cmds: - cmds_by_name[cmd.name] = cmd - - # Get current convert mapping for client - client = SOURCE_CLIENT - mapping = set(process_mapping(os.path.join(root, client))) - - print('* Checking consistency between dispatch tables and vRPCConvertParams') - - # Check mapping consistency - errors = 0 - for (cmdname, argidx, argname) in mapping: - try: - rargnames = cmds_by_name[cmdname].args[argidx].names - except IndexError: - print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname)) - errors += 1 - continue - if argname not in rargnames: - print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr) - errors += 1 - - # Check for conflicts in vRPCConvertParams conversion - # All aliases for an argument must either be present in the - # conversion table, or not. Anything in between means an oversight - # and some aliases won't work. - for cmd in cmds: - for arg in cmd.args: - convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names] - if any(convert) != all(convert): - print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert)) - errors += 1 - arg.convert = all(convert) - - # Check for conversion difference by argument name. - # It is preferable for API consistency that arguments with the same name - # have the same conversion, so bin by argument name. - all_methods_by_argname = defaultdict(list) - converts_by_argname = defaultdict(list) - for cmd in cmds: - for arg in cmd.args: - for argname in arg.names: - all_methods_by_argname[argname].append(cmd.name) - converts_by_argname[argname].append(arg.convert) - - for argname, convert in converts_by_argname.items(): - if all(convert) != any(convert): - if argname in IGNORE_DUMMY_ARGS: - # these are testing or dummy, don't warn for them - continue - print('WARNING: conversion mismatch for argument named %s (%s)' % - (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) - - sys.exit(errors > 0) - - -if __name__ == '__main__': - main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index c4ad00e954..0b15f99448 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -11,6 +11,7 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" "index/txindex -> validation -> index/txindex" + "index/blockfilterindex -> validation -> index/blockfilterindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" @@ -20,6 +21,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "txmempool -> validation -> txmempool" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" + "node/coinstats -> validation -> node/coinstats" ) EXIT_CODE=0 diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 393f734abe..bf7aeb5b4f 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -60,20 +60,11 @@ EXPECTED_BOOST_INCLUDES=( boost/multi_index/ordered_index.hpp boost/multi_index/sequenced_index.hpp boost/multi_index_container.hpp - boost/preprocessor/cat.hpp - boost/preprocessor/stringize.hpp boost/process.hpp boost/signals2/connection.hpp boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp - boost/thread/condition_variable.hpp - boost/thread/mutex.hpp - boost/thread/shared_mutex.hpp - boost/thread/thread.hpp - boost/variant.hpp - boost/variant/apply_visitor.hpp - boost/variant/static_visitor.hpp ) for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh new file mode 100755 index 0000000000..c3b6ff3c98 --- /dev/null +++ b/test/lint/lint-python-dead-code.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Find dead Python code. + +export LC_ALL=C + +if ! command -v vulture > /dev/null; then + echo "Skipping Python dead code linting since vulture is not installed. Install by running \"pip3 install vulture\"" + exit 0 +fi + +# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. +# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. +if ! vulture \ + --min-confidence 100 \ + $(git ls-files -- "*.py"); then + echo "Python dead code detection found some issues" + exit 1 +fi diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index c800fd20db..37fcf7804a 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -33,7 +33,7 @@ if [ -z "${COMMIT_RANGE}" ]; then fi showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then + if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then echo "Failed to get a diff" exit 1 fi diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 29221a94f2..3fc9fac25c 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -13,7 +13,7 @@ mutex:CConnman::ThreadOpenConnections mutex:CConnman::ThreadOpenAddedConnections mutex:CConnman::SocketHandler mutex:UpdateTip -mutex:PeerManager::UpdatedBlockTip +mutex:PeerManagerImpl::UpdatedBlockTip mutex:g_best_block_mutex # race (TODO fix) @@ -28,6 +28,7 @@ race:BerkeleyBatch race:BerkeleyDatabase race:DatabaseBatch race:leveldb::DBImpl::DeleteObsoleteFiles +race:validation_chainstatemanager_tests race:zmq::* race:bitcoin-qt diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 9a52cd4b57..97f0f45e7f 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,3 +1,7 @@ +# -fsanitize=undefined suppressions +# ================================= +# No suppressions at the moment. Hooray! + # -fsanitize=integer suppressions # =============================== # Unsigned integer overflow occurs when the result of an unsigned integer @@ -6,7 +10,8 @@ # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. -unsigned-integer-overflow:*/include/c++/*/bits/basic_string.tcc +unsigned-integer-overflow:*/include/c++/ +unsigned-integer-overflow:addrman.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -15,34 +20,41 @@ unsigned-integer-overflow:bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h +unsigned-integer-overflow:coins.cpp +unsigned-integer-overflow:compressor.cpp unsigned-integer-overflow:core_write.cpp -unsigned-integer-overflow:crypto/* +unsigned-integer-overflow:crypto/ +# unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange +unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:leveldb/db/log_reader.cc -unsigned-integer-overflow:leveldb/util/bloom.cc -unsigned-integer-overflow:leveldb/util/crc32c.h -unsigned-integer-overflow:leveldb/util/hash.cc +unsigned-integer-overflow:leveldb/ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h +unsigned-integer-overflow:pubkey.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp - -implicit-integer-sign-change:*/include/c++/*/bits/*.h +implicit-integer-sign-change:*/include/boost/ +implicit-integer-sign-change:*/include/c++/ implicit-integer-sign-change:*/new_allocator.h -implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp +implicit-integer-sign-change:addrman.h implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp implicit-integer-sign-change:bloom.cpp -implicit-integer-sign-change:chain.* +implicit-integer-sign-change:chain.cpp +implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h -implicit-integer-sign-change:crypto/* +implicit-integer-sign-change:crc32c/ +implicit-integer-sign-change:crypto/ +# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange +implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:key.cpp implicit-integer-sign-change:noui.cpp +implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h implicit-integer-sign-change:protocol.cpp implicit-integer-sign-change:script/bitcoinconsensus.cpp @@ -53,24 +65,38 @@ implicit-integer-sign-change:test/coins_tests.cpp implicit-integer-sign-change:test/pow_tests.cpp implicit-integer-sign-change:test/prevector_tests.cpp implicit-integer-sign-change:test/sighash_tests.cpp +implicit-integer-sign-change:test/skiplist_tests.cpp implicit-integer-sign-change:test/streams_tests.cpp implicit-integer-sign-change:test/transaction_tests.cpp implicit-integer-sign-change:txmempool.cpp -implicit-integer-sign-change:util/strencodings.* +implicit-integer-sign-change:util/strencodings.cpp +implicit-integer-sign-change:util/strencodings.h implicit-integer-sign-change:validation.cpp implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp +implicit-signed-integer-truncation:addrman.cpp +implicit-signed-integer-truncation:addrman.h implicit-signed-integer-truncation:chain.h -implicit-signed-integer-truncation:crypto/* +implicit-signed-integer-truncation:crypto/ implicit-signed-integer-truncation:cuckoocache.h -implicit-signed-integer-truncation:leveldb/* +implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:net.cpp +implicit-signed-integer-truncation:net_processing.cpp implicit-signed-integer-truncation:streams.h implicit-signed-integer-truncation:test/arith_uint256_tests.cpp implicit-signed-integer-truncation:test/skiplist_tests.cpp implicit-signed-integer-truncation:torcontrol.cpp -implicit-unsigned-integer-truncation:crypto/* -implicit-unsigned-integer-truncation:leveldb/* +implicit-unsigned-integer-truncation:*/include/c++/ +implicit-unsigned-integer-truncation:crypto/ +implicit-unsigned-integer-truncation:leveldb/ # std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage -implicit-integer-sign-change:crc32c/* +shift-base:*/include/c++/ +shift-base:arith_uint256.cpp +shift-base:crypto/ +shift-base:hash.cpp +shift-base:leveldb/ +shift-base:net_processing.cpp +shift-base:streams.h +shift-base:util/bip32.cpp |