diff options
134 files changed, 2249 insertions, 1209 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 801164c368..691582239e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -82,10 +82,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [bionic]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_win64.sh" @@ -160,10 +160,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" task: - name: 'macOS 10.14 [gui, no tests] [bionic]' + name: 'macOS 10.14 [gui, no tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_mac.sh" diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4967e675f6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# This is the top-most EditorConfig file. +root = true + +# For all files. +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +# Source code files +[*.{h,cpp,py,sh}] +indent_size = 4 + +# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +[*.yml] +indent_size = 2 + +# Makefiles +[{*.am,Makefile.*.include}] +indent_style = tab + +# Autoconf scripts +[configure.ac] +indent_size = 2 diff --git a/build-aux/m4/ax_gcc_func_attribute.m4 b/build-aux/m4/ax_gcc_func_attribute.m4 deleted file mode 100644 index c788ca9bd4..0000000000 --- a/build-aux/m4/ax_gcc_func_attribute.m4 +++ /dev/null @@ -1,223 +0,0 @@ -# =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_gcc_func_attribute.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_GCC_FUNC_ATTRIBUTE(ATTRIBUTE) -# -# DESCRIPTION -# -# This macro checks if the compiler supports one of GCC's function -# attributes; many other compilers also provide function attributes with -# the same syntax. Compiler warnings are used to detect supported -# attributes as unsupported ones are ignored by default so quieting -# warnings when using this macro will yield false positives. -# -# The ATTRIBUTE parameter holds the name of the attribute to be checked. -# -# If ATTRIBUTE is supported define HAVE_FUNC_ATTRIBUTE_<ATTRIBUTE>. -# -# The macro caches its result in the ax_cv_have_func_attribute_<attribute> -# variable. -# -# The macro currently supports the following function attributes: -# -# alias -# aligned -# alloc_size -# always_inline -# artificial -# cold -# const -# constructor -# constructor_priority for constructor attribute with priority -# deprecated -# destructor -# dllexport -# dllimport -# error -# externally_visible -# flatten -# format -# format_arg -# gnu_inline -# hot -# ifunc -# leaf -# malloc -# noclone -# noinline -# nonnull -# noreturn -# nothrow -# optimize -# pure -# unused -# used -# visibility -# warning -# warn_unused_result -# weak -# weakref -# -# Unsuppored function attributes will be tested with a prototype returning -# an int and not accepting any arguments and the result of the check might -# be wrong or meaningless so use with care. -# -# LICENSE -# -# Copyright (c) 2013 Gabriele Svelto <gabriele.svelto@gmail.com> -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 3 - -AC_DEFUN([AX_GCC_FUNC_ATTRIBUTE], [ - AS_VAR_PUSHDEF([ac_var], [ax_cv_have_func_attribute_$1]) - - AC_CACHE_CHECK([for __attribute__(($1))], [ac_var], [ - AC_LINK_IFELSE([AC_LANG_PROGRAM([ - m4_case([$1], - [alias], [ - int foo( void ) { return 0; } - int bar( void ) __attribute__(($1("foo"))); - ], - [aligned], [ - int foo( void ) __attribute__(($1(32))); - ], - [alloc_size], [ - void *foo(int a) __attribute__(($1(1))); - ], - [always_inline], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [artificial], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [cold], [ - int foo( void ) __attribute__(($1)); - ], - [const], [ - int foo( void ) __attribute__(($1)); - ], - [constructor_priority], [ - int foo( void ) __attribute__((__constructor__(65535/2))); - ], - [constructor], [ - int foo( void ) __attribute__(($1)); - ], - [deprecated], [ - int foo( void ) __attribute__(($1(""))); - ], - [destructor], [ - int foo( void ) __attribute__(($1)); - ], - [dllexport], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [dllimport], [ - int foo( void ) __attribute__(($1)); - ], - [error], [ - int foo( void ) __attribute__(($1(""))); - ], - [externally_visible], [ - int foo( void ) __attribute__(($1)); - ], - [flatten], [ - int foo( void ) __attribute__(($1)); - ], - [format], [ - int foo(const char *p, ...) __attribute__(($1(printf, 1, 2))); - ], - [format_arg], [ - char *foo(const char *p) __attribute__(($1(1))); - ], - [gnu_inline], [ - inline __attribute__(($1)) int foo( void ) { return 0; } - ], - [hot], [ - int foo( void ) __attribute__(($1)); - ], - [ifunc], [ - int my_foo( void ) { return 0; } - static int (*resolve_foo(void))(void) { return my_foo; } - int foo( void ) __attribute__(($1("resolve_foo"))); - ], - [leaf], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [malloc], [ - void *foo( void ) __attribute__(($1)); - ], - [noclone], [ - int foo( void ) __attribute__(($1)); - ], - [noinline], [ - __attribute__(($1)) int foo( void ) { return 0; } - ], - [nonnull], [ - int foo(char *p) __attribute__(($1(1))); - ], - [noreturn], [ - void foo( void ) __attribute__(($1)); - ], - [nothrow], [ - int foo( void ) __attribute__(($1)); - ], - [optimize], [ - __attribute__(($1(3))) int foo( void ) { return 0; } - ], - [pure], [ - int foo( void ) __attribute__(($1)); - ], - [unused], [ - int foo( void ) __attribute__(($1)); - ], - [used], [ - int foo( void ) __attribute__(($1)); - ], - [visibility], [ - int foo_def( void ) __attribute__(($1("default"))); - int foo_hid( void ) __attribute__(($1("hidden"))); - int foo_int( void ) __attribute__(($1("internal"))); - int foo_pro( void ) __attribute__(($1("protected"))); - ], - [warning], [ - int foo( void ) __attribute__(($1(""))); - ], - [warn_unused_result], [ - int foo( void ) __attribute__(($1)); - ], - [weak], [ - int foo( void ) __attribute__(($1)); - ], - [weakref], [ - static int foo( void ) { return 0; } - static int bar( void ) __attribute__(($1("foo"))); - ], - [ - m4_warn([syntax], [Unsupported attribute $1, the test may fail]) - int foo( void ) __attribute__(($1)); - ] - )], []) - ], - dnl GCC doesn't exit with an error if an unknown attribute is - dnl provided but only outputs a warning, so accept the attribute - dnl only if no warning were issued. - [AS_IF([test -s conftest.err], - [AS_VAR_SET([ac_var], [no])], - [AS_VAR_SET([ac_var], [yes])])], - [AS_VAR_SET([ac_var], [no])]) - ]) - - AS_IF([test yes = AS_VAR_GET([ac_var])], - [AC_DEFINE_UNQUOTED(AS_TR_CPP(HAVE_FUNC_ATTRIBUTE_$1), 1, - [Define to 1 if the system has the `$1' function attribute])], []) - - AS_VAR_POPDEF([ac_var]) -]) diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index b0de2ec0bb..6da011d19b 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,9 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 567145fe47..bbc43cfabd 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -16,4 +16,5 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index affaaaa1aa..3eea0a4565 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) export HOST=x86_64-w64-mingw32 -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file" +export DPKG_ADD_ARCH="i386" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false -export RUN_SECURITY_TESTS="true" export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-fuzz-binary --disable-gui-tests --without-boost-process" # Compiler for MinGW-w64 causes false -Wreturn-type warning. # See https://sourceforge.net/p/mingw-w64/bugs/306/ diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 58a8983e6e..82964897e1 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -13,7 +13,7 @@ for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul, echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" - echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b" + echo "( wine \"${b}_orig\" \"\$@\" ) || ( sleep 1 && wine \"${b}_orig\" \"\$@\" )" >> "$b" chmod +x "$b" fi done diff --git a/configure.ac b/configure.ac index 2662716457..62f1c308b7 100644 --- a/configure.ac +++ b/configure.ac @@ -184,10 +184,16 @@ AC_ARG_ENABLE([extended-functional-tests], AC_ARG_ENABLE([fuzz], AS_HELP_STRING([--enable-fuzz], - [enable building of fuzz targets (default no). enabling this will disable all other targets]), + [build for fuzzing (default no). enabling this will disable all other targets and override --{enable,disable}-fuzz-binary]), [enable_fuzz=$enableval], [enable_fuzz=no]) +AC_ARG_ENABLE([fuzz-binary], + AS_HELP_STRING([--enable-fuzz-binary], + [enable building of fuzz binary (default yes).]), + [enable_fuzz_binary=$enableval], + [enable_fuzz_binary=yes]) + AC_ARG_ENABLE([danger_fuzz_link_all], AS_HELP_STRING([--enable-danger-fuzz-link-all], [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]), @@ -807,10 +813,6 @@ if test x$ac_cv_sys_large_files != x && CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -AX_GCC_FUNC_ATTRIBUTE([visibility]) -AX_GCC_FUNC_ATTRIBUTE([dllexport]) -AX_GCC_FUNC_ATTRIBUTE([dllimport]) - if test x$use_glibc_compat != xno; then AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) @@ -854,7 +856,10 @@ if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - AX_CHECK_COMPILE_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) + dnl -fcf-protection used with Clang 7 causes ld to emit warnings: + dnl ld: error: ... <corrupt x86 feature size: 0x8> + dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we wont use the flag in this case. + AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) dnl stack-clash-protection does not work properly when building for Windows. dnl We use the test case from https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 @@ -975,13 +980,13 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [ AC_MSG_RESULT(no)] ) -AC_MSG_CHECKING([for visibility attribute]) -AC_LINK_IFELSE([AC_LANG_SOURCE([ - int foo_def( void ) __attribute__((visibility("default"))); +AC_MSG_CHECKING([for default visibility attribute]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + int foo(void) __attribute__((visibility("default"))); int main(){} ])], [ - AC_DEFINE(HAVE_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) + AC_DEFINE(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) AC_MSG_RESULT(yes) ], [ @@ -992,6 +997,18 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +AC_MSG_CHECKING([for dllexport attribute]) +AC_COMPILE_IFELSE([AC_LANG_SOURCE([ + __declspec(dllexport) int foo(void); + int main(){} + ])], + [ + AC_DEFINE(HAVE_DLLEXPORT_ATTRIBUTE,1,[Define if the dllexport attribute is supported.]) + AC_MSG_RESULT(yes) + ], + [AC_MSG_RESULT(no)] +) + dnl thread_local is currently disabled when building with glibc back compat. dnl Our minimum supported glibc is 2.17, however support for thread_local dnl did not arrive in glibc until 2.18. @@ -1174,12 +1191,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] ) -dnl Check for reduced exports -if test x$use_reduce_exports = xyes; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], - [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) -fi - AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( @@ -1224,7 +1235,7 @@ AC_DEFUN([SUPPRESS_WARNINGS], dnl enable-fuzz should disable all other targets if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets) + AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no @@ -1240,10 +1251,11 @@ if test "x$enable_fuzz" = "xyes"; then use_upnp=no use_natpmp=no use_zmq=no + enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AC_MSG_CHECKING([whether main function is needed]) + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], [AC_MSG_RESULT([no])], @@ -1254,8 +1266,10 @@ if test "x$enable_fuzz" = "xyes"; then #include <cstdint> #include <cstddef> extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* unterminated comment to remove the main function ... - ]],[[]])]) + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) else BITCOIN_QT_INIT @@ -1269,6 +1283,8 @@ else QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi + + CPPFLAGS="$CPPFLAGS -DPROVIDE_MAIN_FUNCTION" fi if test x$enable_wallet != xno; then @@ -1387,9 +1403,11 @@ BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFL BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB" fi +dnl Check for reduced exports if test x$use_reduce_exports = xyes; then - CXXFLAGS="$CXXFLAGS $RE_CXXFLAGS" - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"],, [[$LDFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]]) fi if test x$use_tests = xyes; then @@ -1713,6 +1731,7 @@ AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) @@ -1729,6 +1748,8 @@ AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) +AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) +AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) @@ -1857,6 +1878,7 @@ echo echo "Options used to compile and link:" echo " boost process = $ax_cv_boost_process" echo " multiprocess = $build_multiprocess" +echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "x$enable_wallet" != "xno"; then echo " with sqlite = $use_sqlite" diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 06b15574a7..60acb0d593 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -35,14 +35,14 @@ def setup(): if not os.path.isdir('bitcoin'): subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] elif not args.kvm: - make_image_prog += ['--lxc'] + make_image_prog += ['--lxc', '--disksize', '13000'] subprocess.check_call(make_image_prog) os.chdir(workdir) - if args.is_bionic and not args.kvm and not args.docker: + if args.is_focal and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') sys.exit(0) @@ -176,7 +176,7 @@ def main(): args = parser.parse_args() workdir = os.getcwd() - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index a0ff87b531..b06fc782a3 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-linux-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -15,6 +15,8 @@ packages: - "ca-certificates" - "curl" - "faketime" +- "g++-8" +- "gcc-8" - "git" - "libtool" - "patch" @@ -45,7 +47,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" @@ -107,7 +109,7 @@ script: | BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 done # Faketime for binaries @@ -130,7 +132,7 @@ script: | # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "riscv64-linux-gnu" ] || [ "${i}" = "powerpc64-linux-gnu" ] || [ "${i}" = "powerpc64le-linux-gnu" ]; then + if [ "${i}" = "powerpc64-linux-gnu" ]; then # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 # TODO: remove this when no longer needed HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" @@ -144,7 +146,7 @@ script: | tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 6fcb21f729..3f0c0c3332 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -2,7 +2,7 @@ name: "bitcoin-dmg-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 2a47e90e6e..0dc531df0e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-osx-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -29,6 +29,7 @@ packages: - "python3-setuptools" - "fonts-tuffy" - "xorriso" +- "libtinfo5" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -39,7 +40,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg xorrisofs" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 6bcd126662..c13c24c3cc 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -2,7 +2,7 @@ name: "bitcoin-win-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1edd8b2e81..95cf0185e2 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-win-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -31,7 +31,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g -fno-ident" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 39d3cb9b50..8ea9e00596 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -204,7 +204,7 @@ fi ########################### # CONFIGFLAGS -CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" case "$HOST" in *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; esac diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md new file mode 100644 index 0000000000..7301968461 --- /dev/null +++ b/contrib/message-capture/message-capture-docs.md @@ -0,0 +1,25 @@ +# Per-Peer Message Capture + +## Purpose + +This feature allows for message capture on a per-peer basis. It answers the simple question: "Can I see what messages my node is sending and receiving?" + +## Usage and Functionality + +* Run `bitcoind` with the `-capturemessages` option. +* Look in the `message_capture` folder in your datadir. + * Typically this will be `~/.bitcoin/message_capture`. + * See that there are many folders inside, one for each peer names with its IP address and port. + * Inside each peer's folder there are two `.dat` files: one is for received messages (`msgs_recv.dat`) and the other is for sent messages (`msgs_sent.dat`). +* Run `contrib/message-capture/message-capture-parser.py` with the proper arguments. + * See the `-h` option for help. + * To see all messages, both sent and received, for all peers use: + ``` + ./contrib/message-capture/message-capture-parser.py -o out.json \ + ~/.bitcoin/message_capture/**/*.dat + ``` + * Note: The messages in the given `.dat` files will be interleaved in chronological order. So, giving both received and sent `.dat` files (as above with `*.dat`) will result in all messages being interleaved in chronological order. + * If an output file is not provided (i.e. the `-o` option is not used), then the output prints to `stdout`. +* View the resulting output. + * The output file is `JSON` formatted. + * Suggestion: use `jq` to view the output, with `jq . out.json` diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py new file mode 100755 index 0000000000..9988478f1b --- /dev/null +++ b/contrib/message-capture/message-capture-parser.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Parse message capture binary files. To be used in conjunction with -capturemessages.""" + +import argparse +import os +import shutil +import sys +from io import BytesIO +import json +from pathlib import Path +from typing import Any, List, Optional + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.messages import ser_uint256 # noqa: E402 +from test_framework.p2p import MESSAGEMAP # noqa: E402 + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +# The test framework classes stores hashes as large ints in many cases. +# These are variables of type uint256 in core. +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. +# As such, they are itemized here. +# Any variables with these names that are of type int are actually uint256 variables. +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) +HASH_INTS = [ + "blockhash", + "block_hash", + "hash", + "hashMerkleRoot", + "hashPrevBlock", + "hashstop", + "prev_header", + "sha256", + "stop_hash", +] + +HASH_INT_VECTORS = [ + "hashes", + "headers", + "vHave", + "vHash", +] + + +class ProgressBar: + def __init__(self, total: float): + self.total = total + self.running = 0 + + def set_progress(self, progress: float): + cols = shutil.get_terminal_size()[0] + if cols <= 12: + return + max_blocks = cols - 9 + num_blocks = int(max_blocks * progress) + print('\r[ {}{} ] {:3.0f}%' + .format('#' * num_blocks, + ' ' * (max_blocks - num_blocks), + progress * 100), + end ='') + + def update(self, more: float): + self.running += more + self.set_progress(self.running / self.total) + + +def to_jsonable(obj: Any) -> Any: + if hasattr(obj, "__dict__"): + return obj.__dict__ + elif hasattr(obj, "__slots__"): + ret = {} # type: Any + for slot in obj.__slots__: + val = getattr(obj, slot, None) + if slot in HASH_INTS and isinstance(val, int): + ret[slot] = ser_uint256(val).hex() + elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + ret[slot] = [ser_uint256(a).hex() for a in val] + else: + ret[slot] = to_jsonable(val) + return ret + elif isinstance(obj, list): + return [to_jsonable(a) for a in obj] + elif isinstance(obj, bytes): + return obj.hex() + else: + return obj + + +def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: + with open(path, 'rb') as f_in: + if progress_bar: + bytes_read = 0 + + while True: + if progress_bar: + # Update progress bar + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + bytes_read = f_in.tell() - 1 + + # Read the Header + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + msgtype = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] # type: bytes + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + + # Start converting the message to a dictionary + msg_dict = {} + msg_dict["direction"] = "recv" if recv else "sent" + msg_dict["time"] = time + msg_dict["size"] = length # "size" is less readable here, but more readable in the output + + msg_ser = BytesIO(f_in.read(length)) + + # Determine message type + if msgtype not in MESSAGEMAP: + # Unrecognized message type + try: + msgtype_tmp = msgtype.decode() + if not msgtype_tmp.isprintable(): + raise UnicodeDecodeError + msg_dict["msgtype"] = msgtype_tmp + except UnicodeDecodeError: + msg_dict["msgtype"] = "UNREADABLE" + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unrecognized message type." + messages.append(msg_dict) + print(f"WARNING - Unrecognized message type {msgtype} in {path}", file=sys.stderr) + continue + + # Deserialize the message + msg = MESSAGEMAP[msgtype]() + msg_dict["msgtype"] = msgtype.decode() + + try: + msg.deserialize(msg_ser) + except KeyboardInterrupt: + raise + except Exception: + # Unable to deserialize message body + msg_ser.seek(0, os.SEEK_SET) + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unable to deserialize message." + messages.append(msg_dict) + print(f"WARNING - Unable to deserialize message in {path}", file=sys.stderr) + continue + + # Convert body of message into a jsonable object + if length: + msg_dict["body"] = to_jsonable(msg) + messages.append(msg_dict) + + if progress_bar: + # Update the progress bar to the end of the current file + # in case we exited the loop early + f_in.seek(0, os.SEEK_END) # Go to end of file + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format(sys.argv[0]), + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "capturepaths", + nargs='+', + help="binary message capture files to parse.") + parser.add_argument( + "-o", "--output", + help="output file. If unset print to stdout") + parser.add_argument( + "-n", "--no-progress-bar", + action='store_true', + help="disable the progress bar. Automatically set if the output is not a terminal") + args = parser.parse_args() + capturepaths = [Path.cwd() / Path(capturepath) for capturepath in args.capturepaths] + output = Path.cwd() / Path(args.output) if args.output else False + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() + + messages = [] # type: List[Any] + if use_progress_bar: + total_size = sum(capture.stat().st_size for capture in capturepaths) + progress_bar = ProgressBar(total_size) + else: + progress_bar = None + + for capture in capturepaths: + process_file(str(capture), messages, "recv" in capture.stem, progress_bar) + + messages.sort(key=lambda msg: msg['time']) + + if use_progress_bar: + progress_bar.set_progress(1) + + jsonrep = json.dumps(messages) + if output: + with open(str(output), 'w+', encoding="utf8") as f_out: + f_out.write(jsonrep) + else: + print(jsonrep) + +if __name__ == "__main__": + main() diff --git a/depends/Makefile b/depends/Makefile index 596a46d4a2..016b73a5cb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -265,7 +265,7 @@ install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin14 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 6237734390..ea06952af4 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -111,12 +111,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 Returns various information about the TX mempool. Only supports JSON as output format. -* loaded : (boolean) if the mempool is fully loaded -* size : (numeric) the number of transactions in the TX mempool -* bytes : (numeric) size of the TX mempool in bytes -* usage : (numeric) total TX mempool memory usage -* maxmempool : (numeric) maximum memory usage for the mempool in bytes -* mempoolminfee : (numeric) minimum feerate (BTC per KB) for tx to be accepted +Refer to the `getmempoolinfo` RPC for documentation of the fields. `GET /rest/mempool/contents.json` diff --git a/doc/build-unix.md b/doc/build-unix.md index 5686ac0e9f..150b447e68 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -9,7 +9,7 @@ Note Always use absolute paths to configure and compile Bitcoin Core and the dependencies. For example, when specifying the path of the dependency: - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX + ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures the usage of the absolute path. @@ -166,9 +166,9 @@ miniupnpc https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and turned off by default. See the configure options for UPnP behavior desired: - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime + --without-miniupnpc No UPnP support, miniupnp not required + --disable-upnp-default (the default) UPnP support turned off by default at runtime + --enable-upnp-default UPnP support turned on by default at runtime libnatpmp --------- @@ -177,9 +177,9 @@ libnatpmp from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and turned off by default. See the configure options for NAT-PMP behavior desired: - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime + --without-natpmp No NAT-PMP support, libnatpmp not required + --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime + --enable-natpmp-default NAT-PMP support turned on by default at runtime Berkeley DB ----------- @@ -199,9 +199,9 @@ Boost ----- If you need to build Boost yourself: - sudo su - ./bootstrap.sh - ./bjam install + sudo su + ./bootstrap.sh + ./bjam install Security @@ -212,8 +212,8 @@ This can be disabled with: Hardening Flags: - ./configure --enable-hardening - ./configure --disable-hardening + ./configure --enable-hardening + ./configure --disable-hardening Hardening enables the following features: @@ -228,7 +228,7 @@ Hardening enables the following features: To test that you have built PIE executable, install scanelf, part of paxutils, and use: - scanelf -e ./bitcoin + scanelf -e ./bitcoin The output should contain: @@ -245,8 +245,8 @@ Hardening enables the following features: `scanelf -e ./bitcoin` The output should contain: - STK/REL/PTL - RW- R-- RW- + STK/REL/PTL + RW- R-- RW- The STK RW- means that the stack is readable and writeable but not executable. diff --git a/doc/descriptors.md b/doc/descriptors.md index 63acb9167f..c4fc2a66bf 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -191,7 +191,7 @@ steps, or for dumping wallet descriptors including private key material. In order to easily represent the sets of scripts currently supported by existing Bitcoin Core wallets, a convenience function `combo` is provided, which takes as input a public key, and describes a set of P2PK, -P2PKH, P2WPKH, and P2SH-P2WPH scripts for that key. In case the key is +P2PKH, P2WPKH, and P2SH-P2WPKH scripts for that key. In case the key is uncompressed, the set only includes P2PK and P2PKH scripts. ### Checksums diff --git a/doc/init.md b/doc/init.md index 99aa0a0def..399b819bf4 100644 --- a/doc/init.md +++ b/doc/init.md @@ -53,11 +53,12 @@ Paths All three configurations assume several paths that might need to be adjusted. -Binary: `/usr/bin/bitcoind` -Configuration file: `/etc/bitcoin/bitcoin.conf` -Data directory: `/var/lib/bitcoind` -PID file: `/var/run/bitcoind/bitcoind.pid` (OpenRC and Upstart) or `/run/bitcoind/bitcoind.pid` (systemd) -Lock file: `/var/lock/subsys/bitcoind` (CentOS) + Binary: /usr/bin/bitcoind + Configuration file: /etc/bitcoin/bitcoin.conf + Data directory: /var/lib/bitcoind + PID file: /var/run/bitcoind/bitcoind.pid (OpenRC and Upstart) or + /run/bitcoind/bitcoind.pid (systemd) + Lock file: /var/lock/subsys/bitcoind (CentOS) The PID directory (if applicable) and data directory should both be owned by the bitcoin user and group. It is advised for security reasons to make the @@ -83,10 +84,10 @@ OpenRC). ### macOS -Binary: `/usr/local/bin/bitcoind` -Configuration file: `~/Library/Application Support/Bitcoin/bitcoin.conf` -Data directory: `~/Library/Application Support/Bitcoin` -Lock file: `~/Library/Application Support/Bitcoin/.lock` + Binary: /usr/local/bin/bitcoind + Configuration file: ~/Library/Application Support/Bitcoin/bitcoin.conf + Data directory: ~/Library/Application Support/Bitcoin + Lock file: ~/Library/Application Support/Bitcoin/.lock Installing Service Configuration ----------------------------------- diff --git a/doc/release-notes.md b/doc/release-notes.md index 8f1e03e16b..0f248494c7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -93,6 +93,10 @@ Tools and Utilities Wallet ------ +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + GUI changes ----------- diff --git a/doc/tor.md b/doc/tor.md index 8a2aef2d07..e38ada5bd6 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -21,39 +21,39 @@ information in the debug log about your Tor configuration. The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all outgoing connections, but more is possible. - -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy - server will be used to try to reach .onion addresses as well. - You need to use -noonion or -onion=0 to explicitly disable - outbound access to onion services. - - -onion=ip:port Set the proxy server to use for Tor onion services. You do not - need to set this if it's the same as -proxy. You can use -onion=0 - to explicitly disable access to onion services. - Note: Only the -proxy option sets the proxy for DNS requests; - with -onion they will not route over Tor, so use -proxy if you - have privacy concerns. - - -listen When using -proxy, listening is disabled by default. If you want - to manually configure an onion service (see section 3), you'll - need to enable it explicitly. - - -connect=X When behind a Tor proxy, you can specify .onion addresses instead - -addnode=X of IP addresses or hostnames in these parameters. It requires - -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with - other P2P nodes. - - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - ipv4, ipv6 or onion. If you use this option with values other - than onion you *cannot* disable onion connections; outgoing onion - connections will be enabled when you use -proxy or -onion. Use - -noonion or -onion=0 if you want to be sure there are no outbound - onion connections over the default proxy or your defined -proxy. + -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy + server will be used to try to reach .onion addresses as well. + You need to use -noonion or -onion=0 to explicitly disable + outbound access to onion services. + + -onion=ip:port Set the proxy server to use for Tor onion services. You do not + need to set this if it's the same as -proxy. You can use -onion=0 + to explicitly disable access to onion services. + Note: Only the -proxy option sets the proxy for DNS requests; + with -onion they will not route over Tor, so use -proxy if you + have privacy concerns. + + -listen When using -proxy, listening is disabled by default. If you want + to manually configure an onion service (see section 3), you'll + need to enable it explicitly. + + -connect=X When behind a Tor proxy, you can specify .onion addresses instead + -addnode=X of IP addresses or hostnames in these parameters. It requires + -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with + other P2P nodes. + + -onlynet=onion Make outgoing connections only to .onion addresses. Incoming + connections are not affected by this option. This option can be + specified multiple times to allow multiple network types, e.g. + ipv4, ipv6 or onion. If you use this option with values other + than onion you *cannot* disable onion connections; outgoing onion + connections will be enabled when you use -proxy or -onion. Use + -noonion or -onion=0 if you want to be sure there are no outbound + onion connections over the default proxy or your defined -proxy. In a typical situation, this suffices to run behind a Tor proxy: - ./bitcoind -proxy=127.0.0.1:9050 + ./bitcoind -proxy=127.0.0.1:9050 ## 2. Automatically create a Bitcoin Core onion service @@ -152,57 +152,57 @@ details). You can also manually configure your node to be reachable from the Tor network. Add these lines to your `/etc/tor/torrc` (or equivalent config file): - HiddenServiceDir /var/lib/tor/bitcoin-service/ - HiddenServicePort 8333 127.0.0.1:8334 + HiddenServiceDir /var/lib/tor/bitcoin-service/ + HiddenServicePort 8333 127.0.0.1:8334 The directory can be different of course, but virtual port numbers should be equal to your bitcoind's P2P listen port (8333 by default), and target addresses and ports should be equal to binding address and port for inbound Tor connections (127.0.0.1:8334 by default). - -externalip=X You can tell bitcoin about its publicly reachable addresses using - this option, and this can be an onion address. Given the above - configuration, you can find your onion address in - /var/lib/tor/bitcoin-service/hostname. For connections - coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), onion addresses are given - preference for your node to advertise itself with. - - You can set multiple local addresses with -externalip. The - one that will be rumoured to a particular peer is the most - compatible one and also using heuristics, e.g. the address - with the most incoming connections, etc. - - -listen You'll need to enable listening for incoming connections, as this - is off by default behind a proxy. - - -discover When -externalip is specified, no attempt is made to discover local - IPv4 or IPv6 addresses. If you want to run a dual stack, reachable - from both Tor and IPv4 (or IPv6), you'll need to either pass your - other addresses using -externalip, or explicitly enable -discover. - Note that both addresses of a dual-stack system may be easily - linkable using traffic analysis. + -externalip=X You can tell bitcoin about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in + /var/lib/tor/bitcoin-service/hostname. For connections + coming from unroutable addresses (such as 127.0.0.1, where the + Tor proxy typically runs), onion addresses are given + preference for your node to advertise itself with. + + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + + -listen You'll need to enable listening for incoming connections, as this + is off by default behind a proxy. + + -discover When -externalip is specified, no attempt is made to discover local + IPv4 or IPv6 addresses. If you want to run a dual stack, reachable + from both Tor and IPv4 (or IPv6), you'll need to either pass your + other addresses using -externalip, or explicitly enable -discover. + Note that both addresses of a dual-stack system may be easily + linkable using traffic analysis. In a typical situation, where you're only reachable via Tor, this should suffice: - ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen + ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing your address. To mitigate this, additionally bind the address of your Tor proxy: - ./bitcoind ... -bind=127.0.0.1 + ./bitcoind ... -bind=127.0.0.1 If you don't care too much about hiding your node, and want to be reachable on IPv4 as well, use `discover` instead: - ./bitcoind ... -discover + ./bitcoind ... -discover and open port 8333 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover + ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover ## 4. Privacy recommendations diff --git a/src/Makefile.am b/src/Makefile.am index da033aee8a..869270d1fc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -233,6 +233,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/error.h \ util/fees.h \ + util/getuniquepath.h \ util/golombrice.h \ util/hasher.h \ util/macros.h \ @@ -242,6 +243,7 @@ BITCOIN_CORE_H = \ util/rbf.h \ util/ref.h \ util/settings.h \ + util/sock.h \ util/spanparsing.h \ util/string.h \ util/system.h \ @@ -556,7 +558,9 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/error.cpp \ util/fees.cpp \ + util/getuniquepath.cpp \ util/hasher.cpp \ + util/sock.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e9f9b73abe..e817bb2ee2 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,9 +2,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -if ENABLE_FUZZ +if ENABLE_FUZZ_BINARY noinst_PROGRAMS += test/fuzz/fuzz -else +endif + +if !ENABLE_FUZZ bin_PROGRAMS += test/test_bitcoin endif @@ -50,6 +52,14 @@ FUZZ_SUITE_LD_COMMON = \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) +if USE_UPNP +FUZZ_SUITE_LD_COMMON += $(MINIUPNPC_LIBS) +endif + +if USE_NATPMP +FUZZ_SUITE_LD_COMMON += $(NATPMP_LIBS) +endif + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -114,6 +124,7 @@ BITCOIN_TESTS =\ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ + test/sock_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ @@ -145,10 +156,16 @@ BITCOIN_TESTS += \ wallet/test/ismine_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp +FUZZ_SUITE_LD_COMMON +=\ + $(LIBBITCOIN_WALLET) \ + $(SQLITE_LIBS) \ + $(BDB_LIBS) + if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif + BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ @@ -172,12 +189,12 @@ test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $( if ENABLE_ZMQ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -if ENABLE_FUZZ - FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -278,8 +295,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/txrequest.cpp - -endif # ENABLE_FUZZ +endif # ENABLE_FUZZ_BINARY nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrman.h b/src/addrman.h index 9ac67b7af6..92a5570953 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -335,22 +335,20 @@ public: * * nNew * * nTried * * number of "new" buckets XOR 2**30 - * * all nNew addrinfos in vvNew - * * all nTried addrinfos in vvTried - * * for each bucket: + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: * * number of elements - * * for each element: index + * * for each element: index in the serialized "all new addresses" + * * asmap checksum * * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it * as incompatible. This is necessary because it did not check the version number on * deserialization. * - * Notice that vvTried, mapAddr and vVector are never encoded explicitly; + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; * they are instead reconstructed from the other information. * - * vvNew is serialized, but only used if ADDRMAN_UNKNOWN_BUCKET_COUNT didn't change, - * otherwise it is reconstructed as well. - * * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * @@ -413,13 +411,13 @@ public: } } } - // Store asmap version after bucket entries so that it + // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_version; + uint256 asmap_checksum; if (m_asmap.size() != 0) { - asmap_version = SerializeHash(m_asmap); + asmap_checksum = SerializeHash(m_asmap); } - s << asmap_version; + s << asmap_checksum; } template <typename Stream> @@ -500,47 +498,63 @@ public: nTried -= nLost; // Store positions in the new table buckets to apply later (if possible). - std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing - - for (int bucket = 0; bucket < nUBuckets; bucket++) { - int nSize = 0; - s >> nSize; - for (int n = 0; n < nSize; n++) { - int nIndex = 0; - s >> nIndex; - if (nIndex >= 0 && nIndex < nNew) { - entryToBucket[nIndex] = bucket; + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector<std::pair<int, int>> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); } } } - uint256 supplied_asmap_version; + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum; if (m_asmap.size() != 0) { - supplied_asmap_version = SerializeHash(m_asmap); + supplied_asmap_checksum = SerializeHash(m_asmap); } - uint256 serialized_asmap_version; + uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_version; + s >> serialized_asmap_checksum; } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - int bucket = entryToBucket[n]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && - info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + CAddrInfo& info = mapInfo[entry_index]; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } else { - // In case the new table data cannot be used (format unknown, bucket count wrong or new asmap), + // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); bucket = info.GetNewBucket(nKey, m_asmap); - nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][nUBucketPos] == -1) { - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } } } diff --git a/src/bench/bench.h b/src/bench/bench.h index bafc7f8716..22f06d8cb8 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <util/macros.h> + #include <chrono> #include <functional> #include <map> @@ -12,8 +14,6 @@ #include <vector> #include <bench/nanobench.h> -#include <boost/preprocessor/cat.hpp> -#include <boost/preprocessor/stringize.hpp> /* * Usage: @@ -56,8 +56,8 @@ public: static void RunAll(const Args& args); }; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo"); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index af5a82f69f..9f7d582e8a 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -48,9 +48,8 @@ static void AssembleBlock(benchmark::Bench& bench) LOCK(::cs_main); // Required for ::AcceptToMemoryPool. for (const auto& txr : txs) { - TxValidationState state; - bool ret{::AcceptToMemoryPool(*test_setup.m_node.mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */)}; - assert(ret); + const MempoolAcceptResult res = ::AcceptToMemoryPool(*test_setup.m_node.mempool, txr, false /* bypass_limits */); + assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index fa41208a31..a9a0ec6f5d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -300,9 +300,12 @@ class NetinfoRequestHandler : public BaseRequestHandler { private: static constexpr int8_t UNKNOWN_NETWORK{-1}; - static constexpr uint8_t m_networks_size{3}; - const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion"}}; - std::array<std::array<uint16_t, m_networks_size + 2>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total/block-relay) + static constexpr int8_t NET_I2P{3}; // pos of "i2p" in m_networks + static constexpr uint8_t m_networks_size{4}; + const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion", "i2p"}}; + std::array<std::array<uint16_t, m_networks_size + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) + uint8_t m_block_relay_peers_count{0}; + uint8_t m_manual_peers_count{0}; int8_t NetworkStringToId(const std::string& str) const { for (uint8_t i = 0; i < m_networks_size; ++i) { @@ -316,12 +319,14 @@ private: bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } bool m_is_asmap_on{false}; + bool m_is_i2p_on{false}; size_t m_max_addr_length{0}; - size_t m_max_age_length{4}; + size_t m_max_age_length{3}; size_t m_max_id_length{2}; struct Peer { std::string addr; std::string sub_version; + std::string conn_type; std::string network; std::string age; double min_ping; @@ -333,6 +338,8 @@ private: int id; int mapped_as; int version; + bool is_bip152_hb_from; + bool is_bip152_hb_to; bool is_block_relay; bool is_outbound; bool operator<(const Peer& rhs) const { return std::tie(is_outbound, min_ping) < std::tie(rhs.is_outbound, rhs.min_ping); } @@ -351,6 +358,14 @@ private: const double milliseconds{round(1000 * seconds)}; return milliseconds > 999999 ? "-" : ToString(milliseconds); } + std::string ConnectionTypeForNetinfo(const std::string& conn_type) const + { + if (conn_type == "outbound-full-relay") return "full"; + if (conn_type == "block-relay-only") return "block"; + if (conn_type == "manual" || conn_type == "feeler") return conn_type; + if (conn_type == "addr-fetch") return "addr"; + return ""; + } const UniValue NetinfoHelp() { return std::string{ @@ -379,6 +394,9 @@ private: " type Type of peer connection\n" " \"full\" - full relay, the default\n" " \"block\" - block relay; like full relay but does not relay transactions or addresses\n" + " \"manual\" - peer we manually added using RPC addnode or the -addnode/-connect config options\n" + " \"feeler\" - short-lived connection for testing addresses\n" + " \"addr\" - address fetch; short-lived connection for requesting addresses\n" " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", or \"cjdns\")\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" @@ -386,6 +404,9 @@ private: " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" + " hb High-bandwidth BIP152 compact block relay\n" + " \".\" (to) - we selected the peer as a high-bandwidth peer\n" + " \"*\" (from) - the peer selected us as a high-bandwidth peer\n" " age Duration of connection to the peer, in minutes\n" " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" @@ -393,7 +414,7 @@ private: " address IP address and port of the peer\n" " version Peer version and subversion concatenated, e.g. \"70016/Satoshi:21.0.0/\"\n\n" "* The connection counts table displays the number of peers by direction, network, and the totals\n" - " for each, as well as a column for block relay peers.\n\n" + " for each, as well as two special outbound columns for block relay peers and manual peers.\n\n" "* The local addresses table lists each local address broadcast by the node, the port, and the score.\n\n" "Examples:\n\n" "Connection counts and local addresses only\n" @@ -450,16 +471,16 @@ public: const std::string network{peer["network"].get_str()}; const int8_t network_id{NetworkStringToId(network)}; if (network_id == UNKNOWN_NETWORK) continue; + m_is_i2p_on |= (network_id == NET_I2P); const bool is_outbound{!peer["inbound"].get_bool()}; const bool is_block_relay{!peer["relaytxes"].get_bool()}; + const std::string conn_type{peer["connection_type"].get_str()}; ++m_counts.at(is_outbound).at(network_id); // in/out by network ++m_counts.at(is_outbound).at(m_networks_size); // in/out overall ++m_counts.at(2).at(network_id); // total by network ++m_counts.at(2).at(m_networks_size); // total overall - if (is_block_relay) { - ++m_counts.at(is_outbound).at(m_networks_size + 1); // in/out block-relay - ++m_counts.at(2).at(m_networks_size + 1); // total block-relay - } + if (conn_type == "block-relay-only") ++m_block_relay_peers_count; + if (conn_type == "manual") ++m_manual_peers_count; if (DetailsRequested()) { // Push data for this peer to the peers vector. const int peer_id{peer["id"].get_int()}; @@ -475,7 +496,9 @@ public: const std::string addr{peer["addr"].get_str()}; const std::string age{conn_time == 0 ? "" : ToString((m_time_now - conn_time) / 60)}; const std::string sub_version{peer["subver"].get_str()}; - m_peers.push_back({addr, sub_version, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_block_relay, is_outbound}); + const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; + const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; + m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_bip152_hb_from, is_bip152_hb_to, is_block_relay, is_outbound}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_age_length = std::max(age.length(), m_max_age_length); m_max_id_length = std::max(ToString(peer_id).length(), m_max_id_length); @@ -489,15 +512,15 @@ public: // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("<-> relay net mping ping send recv txn blk %*s ", m_max_age_length, "age"); + result += strprintf("<-> type net mping ping send recv txn blk hb %*s ", m_max_age_length, "age"); if (m_is_asmap_on) result += " asmap "; result += strprintf("%*s %-*s%s\n", m_max_id_length, "id", IsAddressSelected() ? m_max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); for (const Peer& peer : m_peers) { std::string version{ToString(peer.version) + peer.sub_version}; result += strprintf( - "%3s %5s %5s%7s%7s%5s%5s%5s%5s %*s%*i %*s %-*s%s\n", + "%3s %6s %5s%7s%7s%5s%5s%5s%5s %2s %*s%*i %*s %-*s%s\n", peer.is_outbound ? "out" : "in", - peer.is_block_relay ? "block" : "full", + ConnectionTypeForNetinfo(peer.conn_type), peer.network, PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), @@ -505,6 +528,7 @@ public: peer.last_recv == 0 ? "" : ToString(m_time_now - peer.last_recv), peer.last_trxn == 0 ? "" : ToString((m_time_now - peer.last_trxn) / 60), peer.last_blck == 0 ? "" : ToString((m_time_now - peer.last_blck) / 60), + strprintf("%s%s", peer.is_bip152_hb_to ? "." : " ", peer.is_bip152_hb_from ? "*" : " "), m_max_age_length, // variable spacing peer.age, m_is_asmap_on ? 7 : 0, // variable spacing @@ -515,18 +539,27 @@ public: IsAddressSelected() ? peer.addr : "", IsVersionSelected() && version != "0" ? version : ""); } - result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); + result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); } // Report peer connection totals by type. - result += " ipv4 ipv6 onion total block-relay\n"; + result += " ipv4 ipv6 onion"; + if (m_is_i2p_on) result += " i2p"; + result += " total block"; + if (m_manual_peers_count) result += " manual"; const std::array<std::string, 3> rows{{"in", "out", "total"}}; - for (uint8_t i = 0; i < m_networks_size; ++i) { - result += strprintf("%-5s %5i %5i %5i %5i %5i\n", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2), m_counts.at(i).at(m_networks_size), m_counts.at(i).at(m_networks_size + 1)); + for (uint8_t i = 0; i < 3; ++i) { + result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts + if (m_is_i2p_on) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + result += strprintf(" %5i", m_counts.at(i).at(m_networks_size)); // total peers count + if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts + result += strprintf(" %5i", m_block_relay_peers_count); + if (m_manual_peers_count) result += strprintf(" %5i", m_manual_peers_count); + } } // Report local addresses, ports, and scores. - result += "\nLocal addresses"; + result += "\n\nLocal addresses"; const std::vector<UniValue>& local_addrs{networkinfo["localaddresses"].getValues()}; if (local_addrs.empty()) { result += ": n/a\n"; diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 3e8e5fc7bc..b84d909b07 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -33,51 +33,52 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddCommand("info", "Get wallet info", OptionsCategory::COMMANDS); + argsman.AddCommand("create", "Create new wallet file", OptionsCategory::COMMANDS); + argsman.AddCommand("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", OptionsCategory::COMMANDS); + argsman.AddCommand("dump", "Print out all of the wallet key-value records", OptionsCategory::COMMANDS); + argsman.AddCommand("createfromdump", "Create new wallet file from dumped records", OptionsCategory::COMMANDS); } -static bool WalletAppInit(int argc, char* argv[]) +static bool WalletAppInit(ArgsManager& args, int argc, char* argv[]) { - SetupWalletToolArgs(gArgs); + SetupWalletToolArgs(args); std::string error_message; - if (!gArgs.ParseParameters(argc, argv, error_message)) { + if (!args.ParseParameters(argc, argv, error_message)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (argc < 2 || HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; - if (!gArgs.IsArgSet("-version")) { - strUsage += "\n" - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" - "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" - "Usage:\n" - " bitcoin-wallet [options] <command>\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); - } + if (!args.IsArgSet("-version")) { + strUsage += "\n" + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n"; + strUsage += "\n" + args.GetHelpMessage(); + } tfm::format(std::cout, "%s", strUsage); return false; } // check for printtoconsole, allow -debug - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", args.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")); return false; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); return true; } int main(int argc, char* argv[]) { + ArgsManager& args = gArgs; #ifdef WIN32 util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); @@ -85,7 +86,7 @@ int main(int argc, char* argv[]) SetupEnvironment(); RandomInit(); try { - if (!WalletAppInit(argc, argv)) return EXIT_FAILURE; + if (!WalletAppInit(args, argc, argv)) return EXIT_FAILURE; } catch (const std::exception& e) { PrintExceptionContinue(&e, "WalletAppInit()"); return EXIT_FAILURE; @@ -94,33 +95,19 @@ int main(int argc, char* argv[]) return EXIT_FAILURE; } - std::string method {}; - for(int i = 1; i < argc; ++i) { - if (!IsSwitchChar(argv[i][0])) { - if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); - return EXIT_FAILURE; - } - method = argv[i]; - } - } - - if (method.empty()) { + const auto command = args.GetCommand(); + if (!command) { tfm::format(std::cerr, "No method provided. Run `bitcoin-wallet -help` for valid methods.\n"); return EXIT_FAILURE; } - - // A name must be provided when creating a file - if (method == "create" && !gArgs.IsArgSet("-wallet")) { - tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + if (command->args.size() != 0) { + tfm::format(std::cerr, "Error: Additional arguments provided (%s). Methods do not take arguments. Please refer to `-help`.\n", Join(command->args, ", ")); return EXIT_FAILURE; } - std::string name = gArgs.GetArg("-wallet", ""); - ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) { + if (!WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; } ECC_Stop(); diff --git a/src/clientversion.h b/src/clientversion.h index 2da909f829..0ed3f68094 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CLIENTVERSION_H #define BITCOIN_CLIENTVERSION_H +#include <util/macros.h> + #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H @@ -14,13 +16,6 @@ #error Client version information missing: version is not defined by bitcoin-config.h or in any other way #endif -/** - * Converts the parameter X to a string after macro replacement on X has been performed. - * Don't merge these into one macro! - */ -#define STRINGIZE(X) DO_STRINGIZE(X) -#define DO_STRINGIZE(X) #X - //! Copyright string used in Windows .rc files #define COPYRIGHT_STR "2009-" STRINGIZE(COPYRIGHT_YEAR) " " COPYRIGHT_HOLDERS_FINAL diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index fbd14f9325..e5a0d4cb9c 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -17,7 +17,6 @@ namespace { using limb_t = Num3072::limb_t; using double_limb_t = Num3072::double_limb_t; constexpr int LIMB_SIZE = Num3072::LIMB_SIZE; -constexpr int LIMBS = Num3072::LIMBS; /** 2^3072 - 1103717, the largest 3072-bit safe prime number, is used as the modulus. */ constexpr limb_t MAX_PRIME_DIFF = 1103717; @@ -123,7 +122,7 @@ inline void square_n_mul(Num3072& in_out, const int sq, const Num3072& mul) } // namespace -/** Indicates wether d is larger than the modulus. */ +/** Indicates whether d is larger than the modulus. */ bool Num3072::IsOverflow() const { if (this->limbs[0] <= std::numeric_limits<limb_t>::max() - MAX_PRIME_DIFF) return false; @@ -276,18 +275,33 @@ void Num3072::Divide(const Num3072& a) if (this->IsOverflow()) this->FullReduce(); } -Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { - Num3072 out{}; - uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); - unsigned char tmp[BYTE_SIZE]; - ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, BYTE_SIZE); +Num3072::Num3072(const unsigned char (&data)[BYTE_SIZE]) { + for (int i = 0; i < LIMBS; ++i) { + if (sizeof(limb_t) == 4) { + this->limbs[i] = ReadLE32(data + 4 * i); + } else if (sizeof(limb_t) == 8) { + this->limbs[i] = ReadLE64(data + 8 * i); + } + } +} + +void Num3072::ToBytes(unsigned char (&out)[BYTE_SIZE]) { for (int i = 0; i < LIMBS; ++i) { if (sizeof(limb_t) == 4) { - out.limbs[i] = ReadLE32(tmp + 4 * i); + WriteLE32(out + i * 4, this->limbs[i]); } else if (sizeof(limb_t) == 8) { - out.limbs[i] = ReadLE64(tmp + 8 * i); + WriteLE64(out + i * 8, this->limbs[i]); } } +} + +Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { + unsigned char tmp[Num3072::BYTE_SIZE]; + + uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); + ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, Num3072::BYTE_SIZE); + Num3072 out{tmp}; + return out; } @@ -301,14 +315,8 @@ void MuHash3072::Finalize(uint256& out) noexcept m_numerator.Divide(m_denominator); m_denominator.SetToOne(); // Needed to keep the MuHash object valid - unsigned char data[384]; - for (int i = 0; i < LIMBS; ++i) { - if (sizeof(limb_t) == 4) { - WriteLE32(data + i * 4, m_numerator.limbs[i]); - } else if (sizeof(limb_t) == 8) { - WriteLE64(data + i * 8, m_numerator.limbs[i]); - } - } + unsigned char data[Num3072::BYTE_SIZE]; + m_numerator.ToBytes(data); out = (CHashWriter(SER_DISK, 0) << data).GetSHA256(); } diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h index 0c710007c4..c023a8b9d3 100644 --- a/src/crypto/muhash.h +++ b/src/crypto/muhash.h @@ -22,6 +22,7 @@ private: Num3072 GetInverse() const; public: + static constexpr size_t BYTE_SIZE = 384; #ifdef HAVE___INT128 typedef unsigned __int128 double_limb_t; @@ -48,8 +49,10 @@ public: void Divide(const Num3072& a); void SetToOne(); void Square(); + void ToBytes(unsigned char (&out)[BYTE_SIZE]); Num3072() { this->SetToOne(); }; + Num3072(const unsigned char (&data)[BYTE_SIZE]); SERIALIZE_METHODS(Num3072, obj) { @@ -78,7 +81,7 @@ public: * arbitrary subset of the update operations, allowing them to be * efficiently combined later. * - * Muhash does not support checking if an element is already part of the + * MuHash does not support checking if an element is already part of the * set. That is why this class does not enforce the use of a set as the * data it represents because there is no efficient way to do so. * It is possible to add elements more than once and also to remove @@ -91,8 +94,6 @@ public: class MuHash3072 { private: - static constexpr size_t BYTE_SIZE = 384; - Num3072 m_numerator; Num3072 m_denominator; diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 11cf357f3d..151f1a38f1 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -66,7 +66,7 @@ size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_ if (CheckDiskSpace(m_dir, inc_size)) { FILE *file = Open(pos); if (file) { - LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + LogPrint(BCLog::VALIDATION, "Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); AllocateFileRange(file, pos.nPos, inc_size); fclose(file); return inc_size; diff --git a/src/init.cpp b/src/init.cpp index 716c06cd3a..a77e2cf1cb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -454,8 +454,8 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP @@ -519,10 +519,11 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); #ifdef HAVE_THREAD_LOCAL @@ -1040,16 +1041,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Trim requested connection counts, to fit into system limitations // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 - nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind); + nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); + #ifdef USE_POLL int fd_max = nFD; #else int fd_max = FD_SETSIZE; #endif - nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); + nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); - nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); + nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE, nMaxConnections); if (nMaxConnections < nUserMaxConnections) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); diff --git a/src/net.cpp b/src/net.cpp index 76bf7effa4..5fa405a690 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -20,6 +20,7 @@ #include <protocol.h> #include <random.h> #include <scheduler.h> +#include <util/sock.h> #include <util/strencodings.h> #include <util/translation.h> @@ -429,24 +430,26 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Connect bool connected = false; - SOCKET hSocket = INVALID_SOCKET; + std::unique_ptr<Sock> sock; proxyType proxy; if (addrConnect.IsValid()) { bool proxyConnectionFailed = false; if (GetProxy(addrConnect.GetNetwork(), proxy)) { - hSocket = CreateSocket(proxy.proxy); - if (hSocket == INVALID_SOCKET) { + sock = CreateSock(proxy.proxy); + if (!sock) { return nullptr; } - connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), + *sock, nConnectTimeout, proxyConnectionFailed); } else { // no proxy needed (none set for target network) - hSocket = CreateSocket(addrConnect); - if (hSocket == INVALID_SOCKET) { + sock = CreateSock(addrConnect); + if (!sock) { return nullptr; } - connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout, conn_type == ConnectionType::MANUAL); + connected = ConnectSocketDirectly(addrConnect, sock->Get(), nConnectTimeout, + conn_type == ConnectionType::MANUAL); } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to @@ -454,26 +457,26 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addrman.Attempt(addrConnect, fCountFailure); } } else if (pszDest && GetNameProxy(proxy)) { - hSocket = CreateSocket(proxy.proxy); - if (hSocket == INVALID_SOCKET) { + sock = CreateSock(proxy.proxy); + if (!sock) { return nullptr; } std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); bool proxyConnectionFailed; - connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, host, port, *sock, nConnectTimeout, + proxyConnectionFailed); } if (!connected) { - CloseSocket(hSocket); return nullptr; } // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); + CAddress addr_bind = GetBindAddress(sock->Get()); + CNode* pnode = new CNode(id, nLocalServices, sock->Release(), addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); pnode->AddRef(); // We're making a new connection, harvest entropy from the time (and our peer count) @@ -2188,9 +2191,8 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, return false; } - SOCKET hListenSocket = CreateSocket(addrBind); - if (hListenSocket == INVALID_SOCKET) - { + std::unique_ptr<Sock> sock = CreateSock(addrBind); + if (!sock) { strError = strprintf(Untranslated("Error: Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError.original); return false; @@ -2198,21 +2200,21 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. - setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)); + setsockopt(sock->Get(), SOL_SOCKET, SO_REUSEADDR, (sockopt_arg_type)&nOne, sizeof(int)); // some systems don't have IPV6_V6ONLY but are always v6only; others do have the option // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { #ifdef IPV6_V6ONLY - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); + setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_V6ONLY, (sockopt_arg_type)&nOne, sizeof(int)); #endif #ifdef WIN32 int nProtLevel = PROTECTION_LEVEL_UNRESTRICTED; - setsockopt(hListenSocket, IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)); + setsockopt(sock->Get(), IPPROTO_IPV6, IPV6_PROTECTION_LEVEL, (const char*)&nProtLevel, sizeof(int)); #endif } - if (::bind(hListenSocket, (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) + if (::bind(sock->Get(), (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) @@ -2220,21 +2222,19 @@ bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, else strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)"), addrBind.ToString(), NetworkErrorString(nErr)); LogPrintf("%s\n", strError.original); - CloseSocket(hListenSocket); return false; } LogPrintf("Bound to %s\n", addrBind.ToString()); // Listen for incoming connections - if (listen(hListenSocket, SOMAXCONN) == SOCKET_ERROR) + if (listen(sock->Get(), SOMAXCONN) == SOCKET_ERROR) { strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)"), NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError.original); - CloseSocket(hListenSocket); return false; } - vhListenSocket.push_back(ListenSocket(hListenSocket, permissions)); + vhListenSocket.push_back(ListenSocket(sock->Release(), permissions)); return true; } @@ -2879,6 +2879,9 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + } // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; @@ -2894,18 +2897,14 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; - if (pnode->nSendSize > nSendBufferMaxSize) - pnode->fPauseSend = true; + if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) - pnode->vSendMsg.push_back(std::move(msg.data)); + if (nMessageSize) pnode->vSendMsg.push_back(std::move(msg.data)); // If write queue empty, attempt "optimistic write" - if (optimisticSend == true) - nBytesSent = SocketSendData(*pnode); + if (optimisticSend) nBytesSent = SocketSendData(*pnode); } - if (nBytesSent) - RecordBytesSent(nBytesSent); + if (nBytesSent) RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -2948,3 +2947,31 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } + +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming) +{ + // Note: This function captures the message at the time of processing, + // not at socket receive/send time. + // This ensures that the messages are always in order from an application + // layer (processing) perspective. + auto now = GetTime<std::chrono::microseconds>(); + + // Windows folder names can not include a colon + std::string clean_addr = addr.ToString(); + std::replace(clean_addr.begin(), clean_addr.end(), ':', '_'); + + fs::path base_path = GetDataDir() / "message_capture" / clean_addr; + fs::create_directories(base_path); + + fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); + CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + + ser_writedata64(f, now.count()); + f.write(msg_type.data(), msg_type.length()); + for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { + f << '\0'; + } + uint32_t size = data.size(); + ser_writedata32(f, size); + f.write((const char*)data.data(), data.size()); +} @@ -20,6 +20,7 @@ #include <policy/feerate.h> #include <protocol.h> #include <random.h> +#include <span.h> #include <streams.h> #include <sync.h> #include <threadinterrupt.h> @@ -75,6 +76,8 @@ static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; +/** Number of file descriptors required for message capture **/ +static const int NUM_FDS_MESSAGE_CAPTURE = 1; static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; @@ -442,6 +445,7 @@ public: * messages, implying a preference to receive ADDRv2 instead of ADDR ones. */ std::atomic_bool m_wants_addrv2{false}; + /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the // next time DisconnectNodes() runs @@ -1241,6 +1245,9 @@ inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; } +/** Dump binary message to file, with timestamp */ +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming); + struct NodeEvictionCandidate { NodeId id; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b68453759a..4340eb120c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2178,10 +2178,10 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (orphan_it == mapOrphanTransactions.end()) continue; const CTransactionRef porphanTx = orphan_it->second.tx; - TxValidationState state; - std::list<CTransactionRef> removed_txn; + const MempoolAcceptResult result = AcceptToMemoryPool(m_mempool, porphanTx, false /* bypass_limits */); + const TxValidationState& state = result.m_state; - if (AcceptToMemoryPool(m_mempool, state, porphanTx, &removed_txn, false /* bypass_limits */)) { + if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, porphanTx->GetWitnessHash(), m_connman); for (unsigned int i = 0; i < porphanTx->vout.size(); i++) { @@ -2193,7 +2193,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) } } EraseOrphanTx(orphanHash); - for (const CTransactionRef& removedTx : removed_txn) { + for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } break; @@ -2489,6 +2489,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool fRelay = true; vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; + if (nTime < 0) { + nTime = 0; + } nServices = ServiceFlags(nServiceInt); if (!pfrom.IsInboundConn()) { @@ -2745,12 +2748,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // Feature negotiation of wtxidrelay must happen between VERSION and VERACK - // to avoid relay problems from switching after a connection is up. + // BIP339 defines feature negotiation of wtxidrelay, which must happen between + // VERSION and VERACK to avoid relay problems from switching after a connection is up. if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send wtxidrelay message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a wtxidrelay message after VERACK. LogPrint(BCLog::NET, "wtxidrelay received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -2769,10 +2771,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // BIP155 defines feature negotiation of addrv2 and sendaddrv2, which must happen + // between VERSION and VERACK. if (msg_type == NetMsgType::SENDADDRV2) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send SENDADDRV2 message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a SENDADDRV2 message after VERACK. LogPrint(BCLog::NET, "sendaddrv2 received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -3197,10 +3200,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - TxValidationState state; - std::list<CTransactionRef> lRemovedTxn; + const MempoolAcceptResult result = AcceptToMemoryPool(m_mempool, ptx, false /* bypass_limits */); + const TxValidationState& state = result.m_state; - if (AcceptToMemoryPool(m_mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */)) { + if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { m_mempool.check(&::ChainstateActive().CoinsTip()); // As this version of the transaction was acceptable, we can forget about any // requests for it. @@ -3223,7 +3226,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, tx.GetHash().ToString(), m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); - for (const CTransactionRef& removedTx : lRemovedTxn) { + for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); } @@ -4045,14 +4048,12 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + if (pfrom->fPauseSend) return false; std::list<CNetMessage> msgs; { LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; + if (pfrom->vProcessMsg.empty()) return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; @@ -4061,6 +4062,10 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } CNetMessage& msg(msgs.front()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + } + msg.SetVersion(pfrom->GetCommonVersion()); const std::string& msg_type = msg.m_command; diff --git a/src/netbase.cpp b/src/netbase.cpp index 264029d8a2..24188f83c6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -7,13 +7,17 @@ #include <sync.h> #include <tinyformat.h> +#include <util/sock.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/time.h> #include <atomic> #include <cstdint> +#include <functional> #include <limits> +#include <memory> #ifndef WIN32 #include <fcntl.h> @@ -271,14 +275,6 @@ CService LookupNumeric(const std::string& name, int portDefault) return addr; } -struct timeval MillisToTimeval(int64_t nTimeout) -{ - struct timeval timeout; - timeout.tv_sec = nTimeout / 1000; - timeout.tv_usec = (nTimeout % 1000) * 1000; - return timeout; -} - /** SOCKS version */ enum SOCKSVersion: uint8_t { SOCKS4 = 0x04, @@ -336,8 +332,7 @@ enum class IntrRecvError { * @param data The buffer where the read bytes should be stored. * @param len The number of bytes to read into the specified buffer. * @param timeout The total timeout in milliseconds for this read. - * @param hSocket The socket (has to be in non-blocking mode) from which to read - * bytes. + * @param sock The socket (has to be in non-blocking mode) from which to read bytes. * * @returns An IntrRecvError indicating the resulting status of this read. * IntrRecvError::OK only if all of the specified number of bytes were @@ -347,7 +342,7 @@ enum class IntrRecvError { * Sockets can be made non-blocking with SetSocketNonBlocking(const * SOCKET&, bool). */ -static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const SOCKET& hSocket) +static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { int64_t curTime = GetTimeMillis(); int64_t endTime = curTime + timeout; @@ -355,7 +350,7 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c // (in millis) to break off in case of an interruption. const int64_t maxWait = 1000; while (len > 0 && curTime < endTime) { - ssize_t ret = recv(hSocket, (char*)data, len, 0); // Optimistically try the recv first + ssize_t ret = sock.Recv(data, len, 0); // Optimistically try the recv first if (ret > 0) { len -= ret; data += ret; @@ -364,25 +359,10 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c } else { // Other error or blocking int nErr = WSAGetLastError(); if (nErr == WSAEINPROGRESS || nErr == WSAEWOULDBLOCK || nErr == WSAEINVAL) { - if (!IsSelectableSocket(hSocket)) { - return IntrRecvError::NetworkError; - } // Only wait at most maxWait milliseconds at a time, unless // we're approaching the end of the specified total timeout int timeout_ms = std::min(endTime - curTime, maxWait); -#ifdef USE_POLL - struct pollfd pollfd = {}; - pollfd.fd = hSocket; - pollfd.events = POLLIN; - int nRet = poll(&pollfd, 1, timeout_ms); -#else - struct timeval tval = MillisToTimeval(timeout_ms); - fd_set fdset; - FD_ZERO(&fdset); - FD_SET(hSocket, &fdset); - int nRet = select(hSocket + 1, &fdset, nullptr, nullptr, &tval); -#endif - if (nRet == SOCKET_ERROR) { + if (!sock.Wait(std::chrono::milliseconds{timeout_ms}, Sock::RECV)) { return IntrRecvError::NetworkError; } } else { @@ -436,7 +416,7 @@ static std::string Socks5ErrorString(uint8_t err) * @param port The destination port. * @param auth The credentials with which to authenticate with the specified * SOCKS5 proxy. - * @param hSocket The SOCKS5 proxy socket. + * @param sock The SOCKS5 proxy socket. * * @returns Whether or not the operation succeeded. * @@ -446,7 +426,7 @@ static std::string Socks5ErrorString(uint8_t err) * @see <a href="https://www.ietf.org/rfc/rfc1928.txt">RFC1928: SOCKS Protocol * Version 5</a> */ -static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, const SOCKET& hSocket) +static bool Socks5(const std::string& strDest, int port, const ProxyCredentials* auth, const Sock& sock) { IntrRecvError recvr; LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); @@ -464,12 +444,12 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5Init.push_back(0x01); // 1 method identifier follows... vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); + ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { return error("Error sending to proxy"); } uint8_t pchRet1[2]; - if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; } @@ -486,13 +466,13 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); vAuth.push_back(auth->password.size()); vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); + ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { return error("Error sending authentication to proxy"); } LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); uint8_t pchRetA[2]; - if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { return error("Error reading proxy authentication response"); } if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { @@ -512,12 +492,12 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); vSocks5.push_back((port >> 8) & 0xFF); vSocks5.push_back((port >> 0) & 0xFF); - ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); + ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { return error("Error sending to proxy"); } uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { if (recvr == IntrRecvError::Timeout) { /* If a timeout happens here, this effectively means we timed out while connecting * to the remote node. This is very common for Tor, so do not print an @@ -541,16 +521,16 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials uint8_t pchRet3[256]; switch (pchRet2[3]) { - case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, SOCKS5_RECV_TIMEOUT, hSocket); break; - case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, SOCKS5_RECV_TIMEOUT, hSocket); break; + case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, SOCKS5_RECV_TIMEOUT, sock); break; + case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, SOCKS5_RECV_TIMEOUT, sock); break; case SOCKS5Atyp::DOMAINNAME: { - recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, hSocket); + recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); } int nRecv = pchRet3[0]; - recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, hSocket); + recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, sock); break; } default: return error("Error: malformed proxy response"); @@ -558,41 +538,35 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); } - if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { + if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, sock)) != IntrRecvError::OK) { return error("Error reading from proxy"); } LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } -/** - * Try to create a socket file descriptor with specific properties in the - * communications domain (address family) of the specified service. - * - * For details on the desired properties, see the inline comments in the source - * code. - */ -SOCKET CreateSocket(const CService &addrConnect) +std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) { // Create a sockaddr from the specified service. struct sockaddr_storage sockaddr; socklen_t len = sizeof(sockaddr); - if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot create socket for %s: unsupported network\n", addrConnect.ToString()); - return INVALID_SOCKET; + if (!address_family.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { + LogPrintf("Cannot create socket for %s: unsupported network\n", address_family.ToString()); + return nullptr; } // Create a TCP socket in the address family of the specified service. SOCKET hSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); - if (hSocket == INVALID_SOCKET) - return INVALID_SOCKET; + if (hSocket == INVALID_SOCKET) { + return nullptr; + } // Ensure that waiting for I/O on this socket won't result in undefined // behavior. if (!IsSelectableSocket(hSocket)) { CloseSocket(hSocket); LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); - return INVALID_SOCKET; + return nullptr; } #ifdef SO_NOSIGPIPE @@ -608,11 +582,14 @@ SOCKET CreateSocket(const CService &addrConnect) // Set the non-blocking option on the socket. if (!SetSocketNonBlocking(hSocket, true)) { CloseSocket(hSocket); - LogPrintf("CreateSocket: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); + LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); + return nullptr; } - return hSocket; + return std::make_unique<Sock>(hSocket); } +std::function<std::unique_ptr<Sock>(const CService&)> CreateSock = CreateSockTCP; + template<typename... Args> static void LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args) { std::string error_message = tfm::format(fmt, args...); @@ -786,7 +763,7 @@ bool IsProxy(const CNetAddr &addr) { * @param proxy The SOCKS5 proxy. * @param strDest The destination service to which to connect. * @param port The destination port. - * @param hSocket The socket on which to connect to the SOCKS5 proxy. + * @param sock The socket on which to connect to the SOCKS5 proxy. * @param nTimeout Wait this many milliseconds for the connection to the SOCKS5 * proxy to be established. * @param[out] outProxyConnectionFailed Whether or not the connection to the @@ -794,10 +771,10 @@ bool IsProxy(const CNetAddr &addr) { * * @returns Whether or not the operation succeeded. */ -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool& outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType& proxy, const std::string& strDest, int port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed) { // first connect to proxy server - if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout, true)) { + if (!ConnectSocketDirectly(proxy.proxy, sock.Get(), nTimeout, true)) { outProxyConnectionFailed = true; return false; } @@ -806,11 +783,11 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (uint16_t)port, &random_auth, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, &random_auth, sock)) { return false; } } else { - if (!Socks5(strDest, (uint16_t)port, 0, hSocket)) { + if (!Socks5(strDest, (uint16_t)port, 0, sock)) { return false; } } @@ -869,57 +846,6 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) return false; } -#ifdef WIN32 -std::string NetworkErrorString(int err) -{ - wchar_t buf[256]; - buf[0] = 0; - if(FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, - nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - buf, ARRAYSIZE(buf), nullptr)) - { - return strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err); - } - else - { - return strprintf("Unknown error (%d)", err); - } -} -#else -std::string NetworkErrorString(int err) -{ - char buf[256]; - buf[0] = 0; - /* Too bad there are two incompatible implementations of the - * thread-safe strerror. */ - const char *s; -#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ - s = strerror_r(err, buf, sizeof(buf)); -#else /* POSIX variant always returns message in buffer */ - s = buf; - if (strerror_r(err, buf, sizeof(buf))) - buf[0] = 0; -#endif - return strprintf("%s (%d)", s, err); -} -#endif - -bool CloseSocket(SOCKET& hSocket) -{ - if (hSocket == INVALID_SOCKET) - return false; -#ifdef WIN32 - int ret = closesocket(hSocket); -#else - int ret = close(hSocket); -#endif - if (ret) { - LogPrintf("Socket close failed: %d. Error: %s\n", hSocket, NetworkErrorString(WSAGetLastError())); - } - hSocket = INVALID_SOCKET; - return ret != SOCKET_ERROR; -} - bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking) { if (fNonBlocking) { diff --git a/src/netbase.h b/src/netbase.h index ac4cd97673..afc373ef49 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -12,7 +12,10 @@ #include <compat.h> #include <netaddress.h> #include <serialize.h> +#include <util/sock.h> +#include <functional> +#include <memory> #include <stdint.h> #include <string> #include <vector> @@ -51,21 +54,25 @@ bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllo bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); CService LookupNumeric(const std::string& name, int portDefault = 0); bool LookupSubNet(const std::string& strSubnet, CSubNet& subnet); -SOCKET CreateSocket(const CService &addrConnect); + +/** + * Create a TCP socket in the given address family. + * @param[in] address_family The socket is created in the same address family as this address. + * @return pointer to the created Sock object or unique_ptr that owns nothing in case of failure + */ +std::unique_ptr<Sock> CreateSockTCP(const CService& address_family); + +/** + * Socket factory. Defaults to `CreateSockTCP()`, but can be overridden by unit tests. + */ +extern std::function<std::unique_ptr<Sock>(const CService&)> CreateSock; + bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout, bool manual_connection); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool& outProxyConnectionFailed); -/** Return readable error string for a network error code */ -std::string NetworkErrorString(int err); -/** Close socket and set hSocket to INVALID_SOCKET */ -bool CloseSocket(SOCKET& hSocket); +bool ConnectThroughProxy(const proxyType& proxy, const std::string& strDest, int port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(const SOCKET& hSocket, bool fNonBlocking); /** Set the TCP_NODELAY flag on a socket */ bool SetSocketNoDelay(const SOCKET& hSocket); -/** - * Convert milliseconds to a struct timeval for e.g. select. - */ -struct timeval MillisToTimeval(int64_t nTimeout); void InterruptSocks5(bool interrupt); #endif // BITCOIN_NETBASE_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 56113cb731..b994e79391 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -6,6 +6,7 @@ #include <node/coinstats.h> #include <coins.h> +#include <crypto/muhash.h> #include <hash.h> #include <serialize.h> #include <uint256.h> @@ -24,31 +25,47 @@ static uint64_t GetBogoSize(const CScript& scriptPubKey) scriptPubKey.size() /* scriptPubKey */; } -static void ApplyStats(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) { - assert(!outputs.empty()); - ss << hash; - ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); - stats.nTransactions++; - for (const auto& output : outputs) { - ss << VARINT(output.first + 1); - ss << output.second.out.scriptPubKey; - ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + if (it == outputs.begin()) { + ss << hash; + ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); + } + + ss << VARINT(it->first + 1); + ss << it->second.out.scriptPubKey; + ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + + if (it == std::prev(outputs.end())) { + ss << VARINT(0u); } - ss << VARINT(0u); } -static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) {} + +static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) +{ + COutPoint outpoint = COutPoint(hash, it->first); + Coin coin = it->second; + + CDataStream ss(SER_DISK, PROTOCOL_VERSION); + ss << outpoint; + ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase); + ss << coin.out; + muhash.Insert(MakeUCharSpan(ss)); +} + +template <typename T> +static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); stats.nTransactions++; - for (const auto& output : outputs) { + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + ApplyHash(stats, hash_obj, hash, outputs, it); + stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + stats.nTotalAmount += it->second.out.nValue; + stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); } } @@ -104,6 +121,10 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_t CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); return GetUTXOStats(view, stats, ss, interruption_point); } + case(CoinStatsHashType::MUHASH): { + MuHash3072 muhash; + return GetUTXOStats(view, stats, muhash, interruption_point); + } case(CoinStatsHashType::NONE): { return GetUTXOStats(view, stats, nullptr, interruption_point); } @@ -116,10 +137,18 @@ static void PrepareHash(CHashWriter& ss, const CCoinsStats& stats) { ss << stats.hashBlock; } +// MuHash does not need the prepare step +static void PrepareHash(MuHash3072& muhash, CCoinsStats& stats) {} static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) { stats.hashSerialized = ss.GetHash(); } +static void FinalizeHash(MuHash3072& muhash, CCoinsStats& stats) +{ + uint256 out; + muhash.Finalize(out); + stats.hashSerialized = out; +} static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c56bfc2ad..f02b95235f 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -16,6 +16,7 @@ class CCoinsView; enum class CoinStatsHashType { HASH_SERIALIZED, + MUHASH, NONE, }; diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index d3bb9687a8..6f5acf41e3 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -50,22 +50,22 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } if (!node.mempool->exists(hashTx)) { // Transaction is not already in the mempool. - TxValidationState state; if (max_tx_fee > 0) { // First, call ATMP with test_accept and check the fee. If ATMP // fails here, return error immediately. - CAmount fee{0}; - if (!AcceptToMemoryPool(*node.mempool, state, tx, - nullptr /* plTxnReplaced */, false /* bypass_limits */, /* test_accept */ true, &fee)) { - return HandleATMPError(state, err_string); - } else if (fee > max_tx_fee) { + const MempoolAcceptResult result = AcceptToMemoryPool(*node.mempool, tx, false /* bypass_limits */, + true /* test_accept */); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { + return HandleATMPError(result.m_state, err_string); + } else if (result.m_base_fees.value() > max_tx_fee) { return TransactionError::MAX_FEE_EXCEEDED; } } // Try to submit the transaction to the mempool. - if (!AcceptToMemoryPool(*node.mempool, state, tx, - nullptr /* plTxnReplaced */, false /* bypass_limits */)) { - return HandleATMPError(state, err_string); + const MempoolAcceptResult result = AcceptToMemoryPool(*node.mempool, tx, false /* bypass_limits */, + false /* test_accept */); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { + return HandleATMPError(result.m_state, err_string); } // Transaction was accepted to the mempool. diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 376acf0963..0c2bbe9516 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -766,10 +766,10 @@ QString NetworkToQString(Network net) assert(false); } -QString ConnectionTypeToQString(ConnectionType conn_type, bool relay_txes) +QString ConnectionTypeToQString(ConnectionType conn_type) { switch (conn_type) { - case ConnectionType::INBOUND: return relay_txes ? QObject::tr("Inbound Full Relay") : QObject::tr("Inbound Block Relay"); + case ConnectionType::INBOUND: return QObject::tr("Inbound"); case ConnectionType::OUTBOUND_FULL_RELAY: return QObject::tr("Outbound Full Relay"); case ConnectionType::BLOCK_RELAY: return QObject::tr("Outbound Block Relay"); case ConnectionType::MANUAL: return QObject::tr("Outbound Manual"); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index edfb5b13a2..9fb108dff9 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -233,7 +233,7 @@ namespace GUIUtil QString NetworkToQString(Network net); /** Convert enum ConnectionType to QString */ - QString ConnectionTypeToQString(ConnectionType conn_type, bool relay_txes); + QString ConnectionTypeToQString(ConnectionType conn_type); /** Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index a252685d2f..2758ee351a 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -462,7 +462,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty constexpr QChar nonbreaking_hyphen(8209); const std::vector<QString> CONNECTION_TYPE_DOC{ - tr("Inbound Full/Block Relay: initiated by peer"), + tr("Inbound: initiated by peer"), tr("Outbound Full Relay: default"), tr("Outbound Block Relay: does not relay transactions or addresses"), tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") @@ -1120,7 +1120,7 @@ void RPCConsole::updateDetailWidget() ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, stats->nodeStats.fRelayTxes)); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == PF_NONE) { ui->peerPermissions->setText(tr("N/A")); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a83b45ae1d..5be02b1e4e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -17,8 +17,8 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> -#include <policy/fees.h> #include <policy/feerate.h> +#include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> #include <primitives/transaction.h> @@ -156,21 +156,11 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) { - // Serialize passed information without accessing chain state of the active chain! - AssertLockNotHeld(cs_main); // For performance reasons + UniValue result = blockheaderToJSON(tip, blockindex); - UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); - const CBlockIndex* pnext; - int confirmations = ComputeNextBlockAndDepth(tip, blockindex, pnext); - result.pushKV("confirmations", confirmations); result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", block.nVersion); - result.pushKV("versionHex", strprintf("%08x", block.nVersion)); - result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); UniValue txs(UniValue::VARR); if (txDetails) { CBlockUndo blockUndo; @@ -189,18 +179,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn } } result.pushKV("tx", txs); - result.pushKV("time", block.GetBlockTime()); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)block.nNonce); - result.pushKV("bits", strprintf("%08x", block.nBits)); - result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); - result.pushKV("nTx", (uint64_t)blockindex->nTx); - if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); - if (pnext) - result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } @@ -1047,13 +1026,26 @@ static RPCHelpMan pruneblockchain() }; } +CoinStatsHashType ParseHashType(const std::string& hash_type_input) +{ + if (hash_type_input == "hash_serialized_2") { + return CoinStatsHashType::HASH_SERIALIZED; + } else if (hash_type_input == "muhash") { + return CoinStatsHashType::MUHASH; + } else if (hash_type_input == "none") { + return CoinStatsHashType::NONE; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s is not a valid hash_type", hash_type_input)); + } +} + static RPCHelpMan gettxoutsetinfo() { return RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n", { - {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'none'."}, + {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -1063,7 +1055,8 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, - {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, + {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, + {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount"}, }}, @@ -1078,7 +1071,7 @@ static RPCHelpMan gettxoutsetinfo() CCoinsStats stats; ::ChainstateActive().ForceFlushStateToDisk(); - const CoinStatsHashType hash_type = ParseHashType(request.params[0], CoinStatsHashType::HASH_SERIALIZED); + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); NodeContext& node = EnsureNodeContext(request.context); @@ -1091,6 +1084,9 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); } + if (hash_type == CoinStatsHashType::MUHASH) { + ret.pushKV("muhash", stats.hashSerialized.GetHex()); + } ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { @@ -1500,6 +1496,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); + ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); @@ -1520,6 +1517,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritizetransaction"}, {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index b75a7b8d26..38a0bddddb 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -365,13 +365,13 @@ static RPCHelpMan signmessagewithprivkey() static RPCHelpMan setmocktime() { return RPCHelpMan{"setmocktime", - "\nSet the local time to given timestamp (-regtest only)\n", - { - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" - " Pass 0 to go back to using the system time."}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{""}, + "\nSet the local time to given timestamp (-regtest only)\n", + { + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" + "Pass 0 to go back to using the system time."}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { if (!Params().IsMockableChain()) { @@ -386,7 +386,10 @@ static RPCHelpMan setmocktime() LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - int64_t time = request.params[0].get_int64(); + const int64_t time{request.params[0].get_int64()}; + if (time < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime can not be negative: %s.", time)); + } SetMockTime(time); if (request.context.Has<NodeContext>()) { for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ac42404470..784a53e060 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -946,44 +946,35 @@ static RPCHelpMan testmempoolaccept() result_0.pushKV("txid", tx->GetHash().GetHex()); result_0.pushKV("wtxid", tx->GetWitnessHash().GetHex()); - TxValidationState state; - bool test_accept_res; - CAmount fee{0}; - { - LOCK(cs_main); - test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), - nullptr /* plTxnReplaced */, false /* bypass_limits */, /* test_accept */ true, &fee); - } - - // Check that fee does not exceed maximum fee - if (test_accept_res && max_raw_tx_fee && fee > max_raw_tx_fee) { - result_0.pushKV("allowed", false); - result_0.pushKV("reject-reason", "max-fee-exceeded"); - result.push_back(std::move(result_0)); - return result; - } - result_0.pushKV("allowed", test_accept_res); + const MempoolAcceptResult accept_result = WITH_LOCK(cs_main, return AcceptToMemoryPool(mempool, std::move(tx), + false /* bypass_limits */, /* test_accept */ true)); // Only return the fee and vsize if the transaction would pass ATMP. // These can be used to calculate the feerate. - if (test_accept_res) { - result_0.pushKV("vsize", virtual_size); - UniValue fees(UniValue::VOBJ); - fees.pushKV("base", ValueFromAmount(fee)); - result_0.pushKV("fees", fees); + if (accept_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { + const CAmount fee = accept_result.m_base_fees.value(); + // Check that fee does not exceed maximum fee + if (max_raw_tx_fee && fee > max_raw_tx_fee) { + result_0.pushKV("allowed", false); + result_0.pushKV("reject-reason", "max-fee-exceeded"); + } else { + result_0.pushKV("allowed", true); + result_0.pushKV("vsize", virtual_size); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(fee)); + result_0.pushKV("fees", fees); + } + result.push_back(std::move(result_0)); } else { - if (state.IsInvalid()) { - if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { - result_0.pushKV("reject-reason", "missing-inputs"); - } else { - result_0.pushKV("reject-reason", strprintf("%s", state.GetRejectReason())); - } + result_0.pushKV("allowed", false); + const TxValidationState state = accept_result.m_state; + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_0.pushKV("reject-reason", "missing-inputs"); } else { result_0.pushKV("reject-reason", state.GetRejectReason()); } + result.push_back(std::move(result_0)); } - - result.push_back(std::move(result_0)); return result; }, }; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index bfdba5253c..e890c0108a 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,23 +113,6 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } -CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type) -{ - if (param.isNull()) { - return default_type; - } else { - std::string hash_type_input = param.get_str(); - - if (hash_type_input == "hash_serialized_2") { - return CoinStatsHashType::HASH_SERIALIZED; - } else if (hash_type_input == "none") { - return CoinStatsHashType::NONE; - } else { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%d is not a valid hash_type", hash_type_input)); - } - } -} - std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; diff --git a/src/rpc/util.h b/src/rpc/util.h index 444a013ca1..c54ce85f60 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -77,8 +77,6 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); -CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type); - extern CAmount AmountFromValue(const UniValue& value); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index c5dceac848..b6939127e1 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -11,14 +11,12 @@ #if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #if defined(_WIN32) - #if defined(DLL_EXPORT) - #if defined(HAVE_FUNC_ATTRIBUTE_DLLEXPORT) - #define EXPORT_SYMBOL __declspec(dllexport) - #else - #define EXPORT_SYMBOL - #endif + #if defined(HAVE_DLLEXPORT_ATTRIBUTE) + #define EXPORT_SYMBOL __declspec(dllexport) + #else + #define EXPORT_SYMBOL #endif - #elif defined(HAVE_FUNC_ATTRIBUTE_VISIBILITY) + #elif defined(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE) #define EXPORT_SYMBOL __attribute__ ((visibility ("default"))) #endif #elif defined(MSC_VER) && !defined(STATIC_LIBBITCOINCONSENSUS) diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 8afbe9ebed..dba5ce621a 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -106,8 +106,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator std::vector<valtype> vSolutions; whichTypeRet = Solver(scriptPubKey, vSolutions); - switch (whichTypeRet) - { + switch (whichTypeRet) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -173,10 +172,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator // Could not find witnessScript, add to missing sigdata.missing_witness_script = uint256(vSolutions[0]); return false; - - default: - return false; - } + } // no default case, so the compiler can warn about missing cases + assert(false); } static CScript PushAll(const std::vector<valtype>& values) diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 7967c01858..4d882cd1f1 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -45,8 +45,7 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) std::string GetTxnOutputType(TxoutType t) { - switch (t) - { + switch (t) { case TxoutType::NONSTANDARD: return "nonstandard"; case TxoutType::PUBKEY: return "pubkey"; case TxoutType::PUBKEYHASH: return "pubkeyhash"; @@ -182,7 +181,8 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) std::vector<valtype> vSolutions; TxoutType whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TxoutType::PUBKEY) { + switch (whichType) { + case TxoutType::PUBKEY: { CPubKey pubKey(vSolutions[0]); if (!pubKey.IsValid()) return false; @@ -190,26 +190,28 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = PKHash(pubKey); return true; } - else if (whichType == TxoutType::PUBKEYHASH) - { + case TxoutType::PUBKEYHASH: { addressRet = PKHash(uint160(vSolutions[0])); return true; } - else if (whichType == TxoutType::SCRIPTHASH) - { + case TxoutType::SCRIPTHASH: { addressRet = ScriptHash(uint160(vSolutions[0])); return true; - } else if (whichType == TxoutType::WITNESS_V0_KEYHASH) { + } + case TxoutType::WITNESS_V0_KEYHASH: { WitnessV0KeyHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { + } + case TxoutType::WITNESS_V0_SCRIPTHASH: { WitnessV0ScriptHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_UNKNOWN || whichType == TxoutType::WITNESS_V1_TAPROOT) { + } + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V1_TAPROOT: { WitnessUnknown unk; unk.version = vSolutions[0][0]; std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); @@ -217,8 +219,13 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = unk; return true; } - // Multisig txns have more than one address... - return false; + case TxoutType::MULTISIG: + // Multisig txns have more than one address... + case TxoutType::NULL_DATA: + case TxoutType::NONSTANDARD: + return false; + } // no default case, so the compiler can warn about missing cases + assert(false); } bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index ec487aa3ff..e52cd5230c 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <test/util/setup_common.h> #include <util/system.h> +#include <util/getuniquepath.h> #include <boost/test/unit_test.hpp> @@ -69,6 +70,21 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, "")); BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, {})); } + { + fs::path p1 = GetUniquePath(tmpfolder); + fs::path p2 = GetUniquePath(tmpfolder); + fs::path p3 = GetUniquePath(tmpfolder); + + // Ensure that the parent path is always the same. + BOOST_CHECK_EQUAL(tmpfolder, p1.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p2.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p3.parent_path()); + + // Ensure that generated paths are actually different. + BOOST_CHECK(p1 != p2); + BOOST_CHECK(p2 != p3); + BOOST_CHECK(p1 != p3); + } } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END()
\ No newline at end of file diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 74dec6475e..ba5f0c1a75 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -30,8 +30,6 @@ #include <stdint.h> #include <unistd.h> -#include <vector> - #include <test/fuzz/fuzz.h> void initialize_deserialize() @@ -71,7 +69,7 @@ T Deserialize(CDataStream ds) } template <typename T> -void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj, const Optional<int> protocol_version = nullopt) +void DeserializeFromFuzzingInput(FuzzBufferType buffer, T& obj, const Optional<int> protocol_version = nullopt) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); if (protocol_version) { diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index fd87667755..45151258b3 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -13,15 +13,15 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>>& FuzzTargets() +std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>>& FuzzTargets() { - static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>> g_fuzz_targets; + static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>> g_fuzz_targets; return g_fuzz_targets; } -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init) +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden) { - const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init)); + const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init), hidden); Assert(it_ins.second); } @@ -31,6 +31,7 @@ void initialize() { if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { + if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } Assert(false); @@ -59,8 +60,7 @@ static bool read_stdin(std::vector<uint8_t>& data) extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { static const auto& test_one_input = *Assert(g_test_one_input); - const std::vector<uint8_t> input(data, data + size); - test_one_input(input); + test_one_input({data, size}); return 0; } diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 52841e069a..4abc52c15a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -5,29 +5,36 @@ #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H +#include <span.h> + #include <cstdint> #include <functional> #include <string_view> -#include <vector> -using TypeTestOneInput = std::function<void(const std::vector<uint8_t>&)>; +using FuzzBufferType = Span<const uint8_t>; + +using TypeTestOneInput = std::function<void(FuzzBufferType)>; using TypeInitialize = std::function<void()>; +using TypeHidden = bool; -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init); +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden); -inline void FuzzFrameworkEmptyFun() {} +inline void FuzzFrameworkEmptyInitFun() {} #define FUZZ_TARGET(name) \ - FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyFun) - -#define FUZZ_TARGET_INIT(name, init_fun) \ - void name##_fuzz_target(const std::vector<uint8_t>&); \ - struct name##_Before_Main { \ - name##_Before_Main() \ - { \ - FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun); \ - } \ - } const static g_##name##_before_main; \ - void name##_fuzz_target(const std::vector<uint8_t>& buffer) + FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyInitFun) + +#define FUZZ_TARGET_INIT(name, init_fun) \ + FUZZ_TARGET_INIT_HIDDEN(name, init_fun, false) + +#define FUZZ_TARGET_INIT_HIDDEN(name, init_fun, hidden) \ + void name##_fuzz_target(FuzzBufferType); \ + struct name##_Before_Main { \ + name##_Before_Main() \ + { \ + FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun, hidden); \ + } \ + } const static g_##name##_before_main; \ + void name##_fuzz_target(FuzzBufferType buffer) #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 6e9bb47ff6..a42080eb66 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -9,7 +9,6 @@ #include <cassert> #include <cstdint> -#include <netinet/in.h> #include <vector> FUZZ_TARGET(netaddress) diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index e7cc0f5297..442e32d4ca 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -30,14 +30,32 @@ #include <iostream> #include <memory> #include <string> -#include <vector> namespace { const TestingSetup* g_setup; } // namespace +size_t& GetNumMsgTypes() +{ + static size_t g_num_msg_types{0}; + return g_num_msg_types; +} +#define FUZZ_TARGET_MSG(msg_type) \ + struct msg_type##_Count_Before_Main { \ + msg_type##_Count_Before_Main() \ + { \ + ++GetNumMsgTypes(); \ + } \ + } const static g_##msg_type##_count_before_main; \ + FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ + { \ + fuzz_target(buffer, #msg_type); \ + } + void initialize_process_message() { + Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + static const auto testing_setup = MakeFuzzingContext<const TestingSetup>(); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { @@ -46,7 +64,7 @@ void initialize_process_message() SyncWithValidationInterfaceQueue(); } -void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -61,7 +79,7 @@ void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO } CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); - const bool successfully_connected{true}; + const bool successfully_connected{fuzzed_data_provider.ConsumeBool()}; p2p_node.fSuccessfullyConnected = successfully_connected; connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); @@ -87,27 +105,37 @@ void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO } FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_INIT(process_message_addr, initialize_process_message) { fuzz_target(buffer, "addr"); } -FUZZ_TARGET_INIT(process_message_block, initialize_process_message) { fuzz_target(buffer, "block"); } -FUZZ_TARGET_INIT(process_message_blocktxn, initialize_process_message) { fuzz_target(buffer, "blocktxn"); } -FUZZ_TARGET_INIT(process_message_cmpctblock, initialize_process_message) { fuzz_target(buffer, "cmpctblock"); } -FUZZ_TARGET_INIT(process_message_feefilter, initialize_process_message) { fuzz_target(buffer, "feefilter"); } -FUZZ_TARGET_INIT(process_message_filteradd, initialize_process_message) { fuzz_target(buffer, "filteradd"); } -FUZZ_TARGET_INIT(process_message_filterclear, initialize_process_message) { fuzz_target(buffer, "filterclear"); } -FUZZ_TARGET_INIT(process_message_filterload, initialize_process_message) { fuzz_target(buffer, "filterload"); } -FUZZ_TARGET_INIT(process_message_getaddr, initialize_process_message) { fuzz_target(buffer, "getaddr"); } -FUZZ_TARGET_INIT(process_message_getblocks, initialize_process_message) { fuzz_target(buffer, "getblocks"); } -FUZZ_TARGET_INIT(process_message_getblocktxn, initialize_process_message) { fuzz_target(buffer, "getblocktxn"); } -FUZZ_TARGET_INIT(process_message_getdata, initialize_process_message) { fuzz_target(buffer, "getdata"); } -FUZZ_TARGET_INIT(process_message_getheaders, initialize_process_message) { fuzz_target(buffer, "getheaders"); } -FUZZ_TARGET_INIT(process_message_headers, initialize_process_message) { fuzz_target(buffer, "headers"); } -FUZZ_TARGET_INIT(process_message_inv, initialize_process_message) { fuzz_target(buffer, "inv"); } -FUZZ_TARGET_INIT(process_message_mempool, initialize_process_message) { fuzz_target(buffer, "mempool"); } -FUZZ_TARGET_INIT(process_message_notfound, initialize_process_message) { fuzz_target(buffer, "notfound"); } -FUZZ_TARGET_INIT(process_message_ping, initialize_process_message) { fuzz_target(buffer, "ping"); } -FUZZ_TARGET_INIT(process_message_pong, initialize_process_message) { fuzz_target(buffer, "pong"); } -FUZZ_TARGET_INIT(process_message_sendcmpct, initialize_process_message) { fuzz_target(buffer, "sendcmpct"); } -FUZZ_TARGET_INIT(process_message_sendheaders, initialize_process_message) { fuzz_target(buffer, "sendheaders"); } -FUZZ_TARGET_INIT(process_message_tx, initialize_process_message) { fuzz_target(buffer, "tx"); } -FUZZ_TARGET_INIT(process_message_verack, initialize_process_message) { fuzz_target(buffer, "verack"); } -FUZZ_TARGET_INIT(process_message_version, initialize_process_message) { fuzz_target(buffer, "version"); } +FUZZ_TARGET_MSG(addr); +FUZZ_TARGET_MSG(addrv2); +FUZZ_TARGET_MSG(block); +FUZZ_TARGET_MSG(blocktxn); +FUZZ_TARGET_MSG(cfcheckpt); +FUZZ_TARGET_MSG(cfheaders); +FUZZ_TARGET_MSG(cfilter); +FUZZ_TARGET_MSG(cmpctblock); +FUZZ_TARGET_MSG(feefilter); +FUZZ_TARGET_MSG(filteradd); +FUZZ_TARGET_MSG(filterclear); +FUZZ_TARGET_MSG(filterload); +FUZZ_TARGET_MSG(getaddr); +FUZZ_TARGET_MSG(getblocks); +FUZZ_TARGET_MSG(getblocktxn); +FUZZ_TARGET_MSG(getcfcheckpt); +FUZZ_TARGET_MSG(getcfheaders); +FUZZ_TARGET_MSG(getcfilters); +FUZZ_TARGET_MSG(getdata); +FUZZ_TARGET_MSG(getheaders); +FUZZ_TARGET_MSG(headers); +FUZZ_TARGET_MSG(inv); +FUZZ_TARGET_MSG(mempool); +FUZZ_TARGET_MSG(merkleblock); +FUZZ_TARGET_MSG(notfound); +FUZZ_TARGET_MSG(ping); +FUZZ_TARGET_MSG(pong); +FUZZ_TARGET_MSG(sendaddrv2); +FUZZ_TARGET_MSG(sendcmpct); +FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(tx); +FUZZ_TARGET_MSG(verack); +FUZZ_TARGET_MSG(version); +FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 810f0aac92..ef45196671 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -46,7 +46,7 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); - const bool successfully_connected{true}; + const bool successfully_connected{fuzzed_data_provider.ConsumeBool()}; p2p_node.fSuccessfullyConnected = successfully_connected; p2p_node.fPauseSend = false; g_setup->m_node.peerman->InitializeNode(&p2p_node); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 2091ad5d91..8d9a939dab 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -28,12 +28,12 @@ // // (normal build) // $ mkdir dump -// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot --dumptests; done +// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot.py --dumptests; done // $ ... // -// (fuzz test build) +// (libFuzzer build) // $ mkdir dump-min -// $ ./src/test/fuzz/script_assets_test_minimizer -merge=1 dump-min/ dump/ +// $ FUZZ=script_assets_test_minimizer ./src/test/fuzz/fuzz -merge=1 -use_value_profile=1 dump-min/ dump/ // $ (echo -en '[\n'; cat dump-min/* | head -c -2; echo -en '\n]') >script_assets_test.json namespace { @@ -190,7 +190,7 @@ ECCVerifyHandle handle; } // namespace -FUZZ_TARGET(script_assets_test_minimizer) +FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, FuzzFrameworkEmptyInitFun, /* hidden */ true) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 282a2cd8ca..d6bbb17f63 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -78,7 +78,9 @@ FUZZ_TARGET(string) } (void)SanitizeString(random_string_1); (void)SanitizeString(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)); +#ifndef WIN32 (void)ShellEscape(random_string_1); +#endif // WIN32 int port_out; std::string host_out; SplitHostPort(random_string_1, port_out, host_out); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 47b38b6d23..3621702e45 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -54,7 +54,7 @@ FUZZ_TARGET(system) if (args_manager.GetArgFlags(argument_name) != nullopt) { return; } - args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>() & ~ArgsManager::COMMAND, options_category); }, [&] { // Avoid hitting: diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 366385b619..4dc0dd5f51 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -107,6 +107,22 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(scriptHash)); + // TxoutType::WITNESS_V1_TAPROOT + s.clear(); + s << OP_1 << ToByteVector(uint256::ZERO); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V1_TAPROOT); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{1}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ZERO)); + + // TxoutType::WITNESS_UNKNOWN + s.clear(); + s << OP_16 << ToByteVector(uint256::ONE); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_UNKNOWN); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{16}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ONE)); + // TxoutType::NONSTANDARD s.clear(); s << OP_9 << OP_ADD << OP_11 << OP_EQUAL; diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp new file mode 100644 index 0000000000..cc0e6e7057 --- /dev/null +++ b/src/test/sock_tests.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2021-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <compat.h> +#include <test/util/setup_common.h> +#include <util/sock.h> +#include <util/system.h> + +#include <boost/test/unit_test.hpp> + +#include <thread> + +using namespace std::chrono_literals; + +BOOST_FIXTURE_TEST_SUITE(sock_tests, BasicTestingSetup) + +static bool SocketIsClosed(const SOCKET& s) +{ + // Notice that if another thread is running and creates its own socket after `s` has been + // closed, it may be assigned the same file descriptor number. In this case, our test will + // wrongly pretend that the socket is not closed. + int type; + socklen_t len = sizeof(type); + return getsockopt(s, SOL_SOCKET, SO_TYPE, (sockopt_arg_type)&type, &len) == SOCKET_ERROR; +} + +static SOCKET CreateSocket() +{ + const SOCKET s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + BOOST_REQUIRE(s != static_cast<SOCKET>(SOCKET_ERROR)); + return s; +} + +BOOST_AUTO_TEST_CASE(constructor_and_destructor) +{ + const SOCKET s = CreateSocket(); + Sock* sock = new Sock(s); + BOOST_CHECK_EQUAL(sock->Get(), s); + BOOST_CHECK(!SocketIsClosed(s)); + delete sock; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(move_constructor) +{ + const SOCKET s = CreateSocket(); + Sock* sock1 = new Sock(s); + Sock* sock2 = new Sock(std::move(*sock1)); + delete sock1; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_CHECK_EQUAL(sock2->Get(), s); + delete sock2; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(move_assignment) +{ + const SOCKET s = CreateSocket(); + Sock* sock1 = new Sock(s); + Sock* sock2 = new Sock(); + *sock2 = std::move(*sock1); + delete sock1; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_CHECK_EQUAL(sock2->Get(), s); + delete sock2; + BOOST_CHECK(SocketIsClosed(s)); +} + +BOOST_AUTO_TEST_CASE(release) +{ + SOCKET s = CreateSocket(); + Sock* sock = new Sock(s); + BOOST_CHECK_EQUAL(sock->Release(), s); + delete sock; + BOOST_CHECK(!SocketIsClosed(s)); + BOOST_REQUIRE(CloseSocket(s)); +} + +BOOST_AUTO_TEST_CASE(reset) +{ + const SOCKET s = CreateSocket(); + Sock sock(s); + sock.Reset(); + BOOST_CHECK(SocketIsClosed(s)); +} + +#ifndef WIN32 // Windows does not have socketpair(2). + +static void CreateSocketPair(int s[2]) +{ + BOOST_REQUIRE_EQUAL(socketpair(AF_UNIX, SOCK_STREAM, 0, s), 0); +} + +static void SendAndRecvMessage(const Sock& sender, const Sock& receiver) +{ + const char* msg = "abcd"; + constexpr size_t msg_len = 4; + char recv_buf[10]; + + BOOST_CHECK_EQUAL(sender.Send(msg, msg_len, 0), msg_len); + BOOST_CHECK_EQUAL(receiver.Recv(recv_buf, sizeof(recv_buf), 0), msg_len); + BOOST_CHECK_EQUAL(strncmp(msg, recv_buf, msg_len), 0); +} + +BOOST_AUTO_TEST_CASE(send_and_receive) +{ + int s[2]; + CreateSocketPair(s); + + Sock* sock0 = new Sock(s[0]); + Sock* sock1 = new Sock(s[1]); + + SendAndRecvMessage(*sock0, *sock1); + + Sock* sock0moved = new Sock(std::move(*sock0)); + Sock* sock1moved = new Sock(); + *sock1moved = std::move(*sock1); + + delete sock0; + delete sock1; + + SendAndRecvMessage(*sock1moved, *sock0moved); + + delete sock0moved; + delete sock1moved; + + BOOST_CHECK(SocketIsClosed(s[0])); + BOOST_CHECK(SocketIsClosed(s[1])); +} + +BOOST_AUTO_TEST_CASE(wait) +{ + int s[2]; + CreateSocketPair(s); + + Sock sock0(s[0]); + Sock sock1(s[1]); + + std::thread waiter([&sock0]() { sock0.Wait(24h, Sock::RECV); }); + + BOOST_REQUIRE_EQUAL(sock1.Send("a", 1, 0), 1); + + waiter.join(); +} + +#endif /* WIN32 */ + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 7e6246d68f..9b0d2e3135 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -30,25 +30,21 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK(CTransaction(coinbaseTx).IsCoinBase()); - TxValidationState state; - LOCK(cs_main); unsigned int initialPoolSize = m_node.mempool->size(); + const MempoolAcceptResult result = AcceptToMemoryPool(*m_node.mempool, MakeTransactionRef(coinbaseTx), + true /* bypass_limits */); - BOOST_CHECK_EQUAL( - false, - AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(coinbaseTx), - nullptr /* plTxnReplaced */, - true /* bypass_limits */)); + BOOST_CHECK(result.m_result_type == MempoolAcceptResult::ResultType::INVALID); // Check that the transaction hasn't been added to mempool. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); // Check that the validation state reflects the unsuccessful attempt. - BOOST_CHECK(state.IsInvalid()); - BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); - BOOST_CHECK(state.GetResult() == TxValidationResult::TX_CONSENSUS); + BOOST_CHECK(result.m_state.IsInvalid()); + BOOST_CHECK_EQUAL(result.m_state.GetRejectReason(), "coinbase"); + BOOST_CHECK(result.m_state.GetResult() == TxValidationResult::TX_CONSENSUS); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index bed2ba3608..af0090cc10 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -28,9 +28,9 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) const auto ToMemPool = [this](const CMutableTransaction& tx) { LOCK(cs_main); - TxValidationState state; - return AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(tx), - nullptr /* plTxnReplaced */, true /* bypass_limits */); + const MempoolAcceptResult result = AcceptToMemoryPool(*m_node.mempool, MakeTransactionRef(tx), + true /* bypass_limits */); + return result.m_result_type == MempoolAcceptResult::ResultType::VALID; }; // Create a double-spend of mature coinbase txn: diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 4133f2623b..dccbf00d6f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -13,6 +13,7 @@ #include <test/util/setup_common.h> #include <test/util/str.h> #include <uint256.h> +#include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/spanparsing.h> @@ -1816,7 +1817,7 @@ BOOST_AUTO_TEST_CASE(test_DirIsWritable) BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), true); // Should not be able to write to a non-existent dir. - tmpdirname = tmpdirname / fs::unique_path(); + tmpdirname = GetUniquePath(tmpdirname); BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), false); fs::create_directory(tmpdirname); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index ec45d9a434..6c18a9e1bb 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -283,15 +283,9 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Add the txs to the tx pool { LOCK(cs_main); - TxValidationState state; - std::list<CTransactionRef> plTxnReplaced; for (const auto& tx : txs) { - BOOST_REQUIRE(AcceptToMemoryPool( - *m_node.mempool, - state, - tx, - &plTxnReplaced, - /* bypass_limits */ false)); + const MempoolAcceptResult result = AcceptToMemoryPool(*m_node.mempool, tx, false /* bypass_limits */); + BOOST_REQUIRE(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 908ad35e1b..605c77fc3a 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -14,6 +14,7 @@ #include <netbase.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/time.h> #include <deque> #include <functional> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c370f9e981..9fa7b4e251 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -9,14 +9,14 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <optional.h> -#include <validation.h> -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <policy/settings.h> #include <reverse_iterator.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <util/time.h> +#include <validation.h> #include <validationinterface.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, @@ -396,7 +396,10 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} + m_total_fee += entry.GetFee(); + if (minerPolicyEstimator) { + minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + } vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -432,6 +435,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) vTxHashes.clear(); totalTxSize -= it->GetTxSize(); + m_total_fee -= it->GetFee(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); mapTx.erase(it); @@ -590,6 +594,7 @@ void CTxMemPool::_clear() mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; + m_total_fee = 0; cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; @@ -623,6 +628,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; + CAmount check_total_fee{0}; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); @@ -632,6 +638,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->GetTxSize(); + check_total_fee += it->GetFee(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); @@ -726,6 +733,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const } assert(totalTxSize == checkTotal); + assert(m_total_fee == check_total_fee); assert(innerUsage == cachedInnerUsage); } diff --git a/src/txmempool.h b/src/txmempool.h index 0a9cd81ff5..c0df33fe13 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -19,8 +19,8 @@ #include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> -#include <sync.h> #include <random.h> +#include <sync.h> #include <util/hasher.h> #include <boost/multi_index_container.hpp> @@ -478,8 +478,9 @@ private: std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; - uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. - uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) + uint64_t totalTxSize GUARDED_BY(cs); //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. + CAmount m_total_fee GUARDED_BY(cs); //!< sum of all mempool tx's fees (NOT modified fee) + uint64_t cachedInnerUsage GUARDED_BY(cs); //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; @@ -724,6 +725,12 @@ public: return totalTxSize; } + CAmount GetTotalFee() const EXCLUSIVE_LOCKS_REQUIRED(cs) + { + AssertLockHeld(cs); + return m_total_fee; + } + bool exists(const GenTxid& gtxid) const { LOCK(cs); diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp new file mode 100644 index 0000000000..9839d2f624 --- /dev/null +++ b/src/util/getuniquepath.cpp @@ -0,0 +1,10 @@ +#include <random.h> +#include <fs.h> +#include <util/strencodings.h> + +fs::path GetUniquePath(const fs::path& base) +{ + FastRandomContext rnd; + fs::path tmpFile = base / HexStr(rnd.randbytes(8)); + return tmpFile; +}
\ No newline at end of file diff --git a/src/util/getuniquepath.h b/src/util/getuniquepath.h new file mode 100644 index 0000000000..e0c6147876 --- /dev/null +++ b/src/util/getuniquepath.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GETUNIQUEPATH_H +#define BITCOIN_UTIL_GETUNIQUEPATH_H + +#include <fs.h> + +/** + * Helper function for getting a unique path + * + * @param[in] base Base path + * @returns base joined with a random 8-character long string. + * @post Returned path is unique with high probability. + */ +fs::path GetUniquePath(const fs::path& base); + +#endif // BITCOIN_UTIL_GETUNIQUEPATH_H
\ No newline at end of file diff --git a/src/util/macros.h b/src/util/macros.h index 36ea87c0fe..0887c80fd7 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -8,4 +8,11 @@ #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) +/** + * Converts the parameter X to a string after macro replacement on X has been performed. + * Don't merge these into one macro! + */ +#define STRINGIZE(X) DO_STRINGIZE(X) +#define DO_STRINGIZE(X) #X + #endif // BITCOIN_UTIL_MACROS_H diff --git a/src/util/sock.cpp b/src/util/sock.cpp new file mode 100644 index 0000000000..4c65b5b680 --- /dev/null +++ b/src/util/sock.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <compat.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/sock.h> +#include <util/system.h> +#include <util/time.h> + +#include <codecvt> +#include <cwchar> +#include <locale> +#include <string> + +#ifdef USE_POLL +#include <poll.h> +#endif + +Sock::Sock() : m_socket(INVALID_SOCKET) {} + +Sock::Sock(SOCKET s) : m_socket(s) {} + +Sock::Sock(Sock&& other) +{ + m_socket = other.m_socket; + other.m_socket = INVALID_SOCKET; +} + +Sock::~Sock() { Reset(); } + +Sock& Sock::operator=(Sock&& other) +{ + Reset(); + m_socket = other.m_socket; + other.m_socket = INVALID_SOCKET; + return *this; +} + +SOCKET Sock::Get() const { return m_socket; } + +SOCKET Sock::Release() +{ + const SOCKET s = m_socket; + m_socket = INVALID_SOCKET; + return s; +} + +void Sock::Reset() { CloseSocket(m_socket); } + +ssize_t Sock::Send(const void* data, size_t len, int flags) const +{ + return send(m_socket, static_cast<const char*>(data), len, flags); +} + +ssize_t Sock::Recv(void* buf, size_t len, int flags) const +{ + return recv(m_socket, static_cast<char*>(buf), len, flags); +} + +bool Sock::Wait(std::chrono::milliseconds timeout, Event requested) const +{ +#ifdef USE_POLL + pollfd fd; + fd.fd = m_socket; + fd.events = 0; + if (requested & RECV) { + fd.events |= POLLIN; + } + if (requested & SEND) { + fd.events |= POLLOUT; + } + + return poll(&fd, 1, count_milliseconds(timeout)) != SOCKET_ERROR; +#else + if (!IsSelectableSocket(m_socket)) { + return false; + } + + fd_set fdset_recv; + fd_set fdset_send; + FD_ZERO(&fdset_recv); + FD_ZERO(&fdset_send); + + if (requested & RECV) { + FD_SET(m_socket, &fdset_recv); + } + + if (requested & SEND) { + FD_SET(m_socket, &fdset_send); + } + + timeval timeout_struct = MillisToTimeval(timeout); + + return select(m_socket + 1, &fdset_recv, &fdset_send, nullptr, &timeout_struct) != SOCKET_ERROR; +#endif /* USE_POLL */ +} + +#ifdef WIN32 +std::string NetworkErrorString(int err) +{ + wchar_t buf[256]; + buf[0] = 0; + if(FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK, + nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + buf, ARRAYSIZE(buf), nullptr)) + { + return strprintf("%s (%d)", std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf), err); + } + else + { + return strprintf("Unknown error (%d)", err); + } +} +#else +std::string NetworkErrorString(int err) +{ + char buf[256]; + buf[0] = 0; + /* Too bad there are two incompatible implementations of the + * thread-safe strerror. */ + const char *s; +#ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ + s = strerror_r(err, buf, sizeof(buf)); +#else /* POSIX variant always returns message in buffer */ + s = buf; + if (strerror_r(err, buf, sizeof(buf))) + buf[0] = 0; +#endif + return strprintf("%s (%d)", s, err); +} +#endif + +bool CloseSocket(SOCKET& hSocket) +{ + if (hSocket == INVALID_SOCKET) + return false; +#ifdef WIN32 + int ret = closesocket(hSocket); +#else + int ret = close(hSocket); +#endif + if (ret) { + LogPrintf("Socket close failed: %d. Error: %s\n", hSocket, NetworkErrorString(WSAGetLastError())); + } + hSocket = INVALID_SOCKET; + return ret != SOCKET_ERROR; +} diff --git a/src/util/sock.h b/src/util/sock.h new file mode 100644 index 0000000000..26fe60f18f --- /dev/null +++ b/src/util/sock.h @@ -0,0 +1,118 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SOCK_H +#define BITCOIN_UTIL_SOCK_H + +#include <compat.h> + +#include <chrono> +#include <string> + +/** + * RAII helper class that manages a socket. Mimics `std::unique_ptr`, but instead of a pointer it + * contains a socket and closes it automatically when it goes out of scope. + */ +class Sock +{ +public: + /** + * Default constructor, creates an empty object that does nothing when destroyed. + */ + Sock(); + + /** + * Take ownership of an existent socket. + */ + explicit Sock(SOCKET s); + + /** + * Copy constructor, disabled because closing the same socket twice is undesirable. + */ + Sock(const Sock&) = delete; + + /** + * Move constructor, grab the socket from another object and close ours (if set). + */ + Sock(Sock&& other); + + /** + * Destructor, close the socket or do nothing if empty. + */ + virtual ~Sock(); + + /** + * Copy assignment operator, disabled because closing the same socket twice is undesirable. + */ + Sock& operator=(const Sock&) = delete; + + /** + * Move assignment operator, grab the socket from another object and close ours (if set). + */ + virtual Sock& operator=(Sock&& other); + + /** + * Get the value of the contained socket. + * @return socket or INVALID_SOCKET if empty + */ + virtual SOCKET Get() const; + + /** + * Get the value of the contained socket and drop ownership. It will not be closed by the + * destructor after this call. + * @return socket or INVALID_SOCKET if empty + */ + virtual SOCKET Release(); + + /** + * Close if non-empty. + */ + virtual void Reset(); + + /** + * send(2) wrapper. Equivalent to `send(this->Get(), data, len, flags);`. Code that uses this + * wrapper can be unit-tested if this method is overridden by a mock Sock implementation. + */ + virtual ssize_t Send(const void* data, size_t len, int flags) const; + + /** + * recv(2) wrapper. Equivalent to `recv(this->Get(), buf, len, flags);`. Code that uses this + * wrapper can be unit-tested if this method is overridden by a mock Sock implementation. + */ + virtual ssize_t Recv(void* buf, size_t len, int flags) const; + + using Event = uint8_t; + + /** + * If passed to `Wait()`, then it will wait for readiness to read from the socket. + */ + static constexpr Event RECV = 0b01; + + /** + * If passed to `Wait()`, then it will wait for readiness to send to the socket. + */ + static constexpr Event SEND = 0b10; + + /** + * Wait for readiness for input (recv) or output (send). + * @param[in] timeout Wait this much for at least one of the requested events to occur. + * @param[in] requested Wait for those events, bitwise-or of `RECV` and `SEND`. + * @return true on success and false otherwise + */ + virtual bool Wait(std::chrono::milliseconds timeout, Event requested) const; + +private: + /** + * Contained socket. `INVALID_SOCKET` designates the object is empty. + */ + SOCKET m_socket; +}; + +/** Return readable error string for a network error code */ +std::string NetworkErrorString(int err); + +/** Close socket and set hSocket to INVALID_SOCKET */ +bool CloseSocket(SOCKET& hSocket); + +#endif // BITCOIN_UTIL_SOCK_H diff --git a/src/util/system.cpp b/src/util/system.cpp index d1fb921642..9a2e719bbc 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,7 +3,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <sync.h> #include <util/system.h> #ifdef HAVE_BOOST_PROCESS @@ -11,6 +10,9 @@ #endif // HAVE_BOOST_PROCESS #include <chainparamsbase.h> +#include <sync.h> +#include <util/check.h> +#include <util/getuniquepath.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -123,7 +125,7 @@ void ReleaseDirectoryLocks() bool DirIsWritable(const fs::path& directory) { - fs::path tmpFile = directory / fs::unique_path(); + fs::path tmpFile = GetUniquePath(directory); FILE* file = fsbridge::fopen(tmpFile, "a"); if (!file) return false; @@ -310,8 +312,22 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin key[0] = '-'; #endif - if (key[0] != '-') + if (key[0] != '-') { + if (!m_accept_any_command && m_command.empty()) { + // The first non-dash arg is a registered command + Optional<unsigned int> flags = GetArgFlags(key); + if (!flags || !(*flags & ArgsManager::COMMAND)) { + error = strprintf("Invalid command '%s'", argv[i]); + return false; + } + } + m_command.push_back(key); + while (++i < argc) { + // The remaining args are command args + m_command.push_back(argv[i]); + } break; + } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') @@ -359,6 +375,26 @@ Optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) const return nullopt; } +std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const +{ + Command ret; + LOCK(cs_args); + auto it = m_command.begin(); + if (it == m_command.end()) { + // No command was passed + return std::nullopt; + } + if (!m_accept_any_command) { + // The registered command + ret.command = *(it++); + } + while (it != m_command.end()) { + // The unregistered command and args (if any) + ret.args.push_back(*(it++)); + } + return ret; +} + std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; @@ -504,8 +540,22 @@ void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strV m_settings.forced_settings[SettingName(strArg)] = strValue; } +void ArgsManager::AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat) +{ + Assert(cmd.find('=') == std::string::npos); + Assert(cmd.at(0) != '-'); + + LOCK(cs_args); + m_accept_any_command = false; // latch to false + std::map<std::string, Arg>& arg_map = m_available_args[cat]; + auto ret = arg_map.emplace(cmd, Arg{"", help, ArgsManager::COMMAND}); + Assert(ret.second); // Fail on duplicate commands +} + void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) { + Assert((flags & ArgsManager::COMMAND) == 0); // use AddCommand + // Split arg name from its help param size_t eq_index = name.find('='); if (eq_index == std::string::npos) { diff --git a/src/util/system.h b/src/util/system.h index d06c30bfa7..5959bc4196 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -166,7 +166,7 @@ struct SectionInfo class ArgsManager { public: - enum Flags { + enum Flags : uint32_t { // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 ALLOW_BOOL = 0x01, ALLOW_INT = 0x02, @@ -181,6 +181,7 @@ public: NETWORK_ONLY = 0x200, // This argument's value is sensitive (such as a password). SENSITIVE = 0x400, + COMMAND = 0x800, }; protected: @@ -193,9 +194,11 @@ protected: mutable RecursiveMutex cs_args; util::Settings m_settings GUARDED_BY(cs_args); + std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); + bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); [[nodiscard]] bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); @@ -246,6 +249,20 @@ public: */ const std::list<SectionInfo> GetUnrecognizedSections() const; + struct Command { + /** The command (if one has been registered with AddCommand), or empty */ + std::string command; + /** + * If command is non-empty: Any args that followed it + * If command is empty: The unregistered command and any args that followed it + */ + std::vector<std::string> args; + }; + /** + * Get the command and command args (returns std::nullopt if no command provided) + */ + std::optional<const Command> GetCommand() const; + /** * Return a vector of strings of the given argument * @@ -332,6 +349,11 @@ public: void AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat); /** + * Add subcommand + */ + void AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat); + + /** * Add many hidden arguments */ void AddHiddenArgs(const std::vector<std::string>& args); diff --git a/src/util/time.cpp b/src/util/time.cpp index e96972fe12..295806c54a 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -7,8 +7,11 @@ #include <config/bitcoin-config.h> #endif +#include <compat.h> #include <util/time.h> +#include <util/check.h> + #include <atomic> #include <boost/date_time/posix_time/posix_time.hpp> #include <ctime> @@ -18,7 +21,7 @@ void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } -static std::atomic<int64_t> nMockTime(0); //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For testing int64_t GetTime() { @@ -46,6 +49,7 @@ template std::chrono::microseconds GetTime(); void SetMockTime(int64_t nMockTimeIn) { + Assert(nMockTimeIn >= 0); nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } @@ -114,3 +118,16 @@ int64_t ParseISO8601DateTime(const std::string& str) return 0; return (ptime - epoch).total_seconds(); } + +struct timeval MillisToTimeval(int64_t nTimeout) +{ + struct timeval timeout; + timeout.tv_sec = nTimeout / 1000; + timeout.tv_usec = (nTimeout % 1000) * 1000; + return timeout; +} + +struct timeval MillisToTimeval(std::chrono::milliseconds ms) +{ + return MillisToTimeval(count_milliseconds(ms)); +} diff --git a/src/util/time.h b/src/util/time.h index c69f604dc6..03b75b5be5 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_UTIL_TIME_H #define BITCOIN_UTIL_TIME_H +#include <compat.h> + #include <chrono> #include <stdint.h> #include <string> @@ -25,6 +27,7 @@ void UninterruptibleSleep(const std::chrono::microseconds& n); * interface that doesn't support std::chrono (e.g. RPC, debug log, or the GUI) */ inline int64_t count_seconds(std::chrono::seconds t) { return t.count(); } +inline int64_t count_milliseconds(std::chrono::milliseconds t) { return t.count(); } inline int64_t count_microseconds(std::chrono::microseconds t) { return t.count(); } /** @@ -57,4 +60,14 @@ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); int64_t ParseISO8601DateTime(const std::string& str); +/** + * Convert milliseconds to a struct timeval for e.g. select. + */ +struct timeval MillisToTimeval(int64_t nTimeout); + +/** + * Convert milliseconds to a struct timeval for e.g. select. + */ +struct timeval MillisToTimeval(std::chrono::milliseconds ms); + #endif // BITCOIN_UTIL_TIME_H diff --git a/src/validation.cpp b/src/validation.cpp index 38df71b994..778d75ce18 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -380,10 +380,8 @@ static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransact auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions - TxValidationState stateDummy; if (!fAddToMempool || (*it)->IsCoinBase() || - !AcceptToMemoryPool(mempool, stateDummy, *it, - nullptr /* plTxnReplaced */, true /* bypass_limits */)) { + AcceptToMemoryPool(mempool, *it, true /* bypass_limits */).m_result_type != MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); @@ -463,9 +461,7 @@ public: // around easier. struct ATMPArgs { const CChainParams& m_chainparams; - TxValidationState &m_state; const int64_t m_accept_time; - std::list<CTransactionRef>* m_replaced_transactions; const bool m_bypass_limits; /* * Return any outpoints which were not previously present in the coins @@ -476,11 +472,10 @@ public: */ std::vector<COutPoint>& m_coins_to_uncache; const bool m_test_accept; - CAmount* m_fee_out; }; // Single transaction acceptance - bool AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + MempoolAcceptResult AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); private: // All the intermediate state that gets passed between the various levels @@ -491,14 +486,17 @@ private: CTxMemPool::setEntries m_all_conflicting; CTxMemPool::setEntries m_ancestors; std::unique_ptr<CTxMemPoolEntry> m_entry; + std::list<CTransactionRef> m_replaced_transactions; bool m_replacement_transaction; + CAmount m_base_fees; CAmount m_modified_fees; CAmount m_conflicting_fees; size_t m_conflicting_size; const CTransactionRef& m_ptx; const uint256& m_hash; + TxValidationState m_state; }; // Run the policy checks on a given transaction, excluding any script checks. @@ -509,18 +507,18 @@ private: // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size // limiting is performed, false otherwise. - bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool Finalize(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) @@ -558,12 +556,12 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256& hash = ws.m_hash; // Copy/alias what we need out of args - TxValidationState &state = args.m_state; const int64_t nAcceptTime = args.m_accept_time; const bool bypass_limits = args.m_bypass_limits; std::vector<COutPoint>& coins_to_uncache = args.m_coins_to_uncache; // Alias what we need out of ws + TxValidationState& state = ws.m_state; std::set<uint256>& setConflicts = ws.m_conflicts; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; CTxMemPool::setEntries& setAncestors = ws.m_ancestors; @@ -683,16 +681,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckSequenceLocks(m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); - CAmount nFees = 0; - if (!Consensus::CheckTxInputs(tx, state, m_view, g_chainman.m_blockman.GetSpendHeight(m_view), nFees)) { + if (!Consensus::CheckTxInputs(tx, state, m_view, g_chainman.m_blockman.GetSpendHeight(m_view), ws.m_base_fees)) { return false; // state filled in by CheckTxInputs } - // If fee_out is passed, return the fee to the caller - if (args.m_fee_out) { - *args.m_fee_out = nFees; - } - // Check for non-standard pay-to-script-hash in inputs const auto& params = args.m_chainparams.GetConsensus(); auto taproot_state = VersionBitsState(::ChainActive().Tip(), params, Consensus::DEPLOYMENT_TAPROOT, versionbitscache); @@ -707,7 +699,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); // nModifiedFees includes any fee deltas from PrioritiseTransaction - nModifiedFees = nFees; + nModifiedFees = ws.m_base_fees; m_pool.ApplyDelta(hash, nModifiedFees); // Keep track of transactions that spend a coinbase, which we re-scan @@ -721,7 +713,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } - entry.reset(new CTxMemPoolEntry(ptx, nFees, nAcceptTime, ::ChainActive().Height(), + entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, ::ChainActive().Height(), fSpendsCoinbase, nSigOpsCost, lp)); unsigned int nSize = entry->GetTxSize(); @@ -925,11 +917,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; - - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; @@ -952,12 +943,11 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, Prec return true; } -bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; const CChainParams& chainparams = args.m_chainparams; // Check again against the current block tip's script verification @@ -984,11 +974,11 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, P return true; } -bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) +bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - TxValidationState &state = args.m_state; + TxValidationState& state = ws.m_state; const bool bypass_limits = args.m_bypass_limits; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; @@ -1007,8 +997,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) hash.ToString(), FormatMoney(nModifiedFees - nConflictingFees), (int)entry->GetTxSize() - (int)nConflictingSize); - if (args.m_replaced_transactions) - args.m_replaced_transactions->push_back(it->GetSharedTx()); + ws.m_replaced_transactions.push_back(it->GetSharedTx()); } m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); @@ -1031,14 +1020,14 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) +MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) { AssertLockHeld(cs_main); LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) - Workspace workspace(ptx); + Workspace ws(ptx); - if (!PreChecks(args, workspace)) return false; + if (!PreChecks(args, ws)) return MempoolAcceptResult(ws.m_state); // Only compute the precomputed transaction data if we need to verify // scripts (ie, other policy checks pass). We perform the inexpensive @@ -1046,31 +1035,35 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs // checks pass, to mitigate CPU exhaustion denial-of-service attacks. PrecomputedTransactionData txdata; - if (!PolicyScriptChecks(args, workspace, txdata)) return false; + if (!PolicyScriptChecks(args, ws, txdata)) return MempoolAcceptResult(ws.m_state); - if (!ConsensusScriptChecks(args, workspace, txdata)) return false; + if (!ConsensusScriptChecks(args, ws, txdata)) return MempoolAcceptResult(ws.m_state); // Tx was accepted, but not added - if (args.m_test_accept) return true; + if (args.m_test_accept) { + return MempoolAcceptResult(std::move(ws.m_replaced_transactions), ws.m_base_fees); + } - if (!Finalize(args, workspace)) return false; + if (!Finalize(args, ws)) return MempoolAcceptResult(ws.m_state); GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); - return true; + return MempoolAcceptResult(std::move(ws.m_replaced_transactions), ws.m_base_fees); } } // anon namespace /** (try to) add transaction to memory pool with a specified acceptance time **/ -static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, + const CTransactionRef &tx, int64_t nAcceptTime, + bool bypass_limits, bool test_accept) + EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, coins_to_uncache, test_accept, fee_out }; - bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); - if (!res) { + MemPoolAccept::ATMPArgs args { chainparams, nAcceptTime, bypass_limits, coins_to_uncache, test_accept }; + + const MempoolAcceptResult result = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { // Remove coins that were not present in the coins cache before calling ATMPW; // this is to prevent memory DoS in case we receive a large number of // invalid transactions that attempt to overrun the in-memory coins cache @@ -1082,15 +1075,12 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits BlockValidationState state_dummy; ::ChainstateActive().FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); - return res; + return result; } -bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept, CAmount* fee_out) +MempoolAcceptResult AcceptToMemoryPool(CTxMemPool& pool, const CTransactionRef &tx, bool bypass_limits, bool test_accept) { - const CChainParams& chainparams = Params(); - return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, test_accept, fee_out); + return AcceptToMemoryPoolWithTime(Params(), pool, tx, GetTime(), bypass_limits, test_accept); } CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) @@ -3219,7 +3209,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n if ((int)nFile != nLastBlockFile) { if (!fKnown) { - LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; @@ -5029,13 +5019,10 @@ bool LoadMempool(CTxMemPool& pool) if (amountdelta) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } - TxValidationState state; if (nTime > nNow - nExpiryTimeout) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nTime, - nullptr /* plTxnReplaced */, false /* bypass_limits */, - false /* test_accept */); - if (state.IsValid()) { + if (AcceptToMemoryPoolWithTime(chainparams, pool, tx, nTime, false /* bypass_limits */, + false /* test_accept */).m_result_type == MempoolAcceptResult::ResultType::VALID) { ++count; } else { // mempool may contain the transaction already, e.g. from @@ -5159,7 +5146,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin } Optional<uint256> ChainstateManager::SnapshotBlockhash() const { - LOCK(::cs_main); // for m_active_chainstate access + LOCK(::cs_main); if (m_active_chainstate != nullptr) { // If a snapshot chainstate exists, it will always be our active. return m_active_chainstate->m_from_snapshot_blockhash; @@ -5169,6 +5156,7 @@ Optional<uint256> ChainstateManager::SnapshotBlockhash() const { std::vector<CChainState*> ChainstateManager::GetAll() { + LOCK(::cs_main); std::vector<CChainState*> out; if (!IsSnapshotValidated() && m_ibd_chainstate) { @@ -5213,11 +5201,13 @@ CChainState& ChainstateManager::ActiveChainstate() const bool ChainstateManager::IsSnapshotActive() const { - return m_snapshot_chainstate && WITH_LOCK(::cs_main, return m_active_chainstate) == m_snapshot_chainstate.get(); + LOCK(::cs_main); + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } CChainState& ChainstateManager::ValidatedChainstate() const { + LOCK(::cs_main); if (m_snapshot_chainstate && IsSnapshotValidated()) { return *m_snapshot_chainstate.get(); } @@ -5227,6 +5217,7 @@ CChainState& ChainstateManager::ValidatedChainstate() const bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const { + LOCK(::cs_main); return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); } @@ -5242,12 +5233,10 @@ void ChainstateManager::Unload() void ChainstateManager::Reset() { + LOCK(::cs_main); m_ibd_chainstate.reset(); m_snapshot_chainstate.reset(); - { - LOCK(::cs_main); - m_active_chainstate = nullptr; - } + m_active_chainstate = nullptr; m_snapshot_validated = false; } diff --git a/src/validation.h b/src/validation.h index fc7add85b7..238d6009b4 100644 --- a/src/validation.h +++ b/src/validation.h @@ -12,6 +12,7 @@ #include <amount.h> #include <coins.h> +#include <consensus/validation.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> #include <optional.h> @@ -23,6 +24,7 @@ #include <txdb.h> #include <versionbits.h> #include <serialize.h> +#include <util/check.h> #include <util/hasher.h> #include <atomic> @@ -46,7 +48,6 @@ class CConnman; class CScriptCheck; class CTxMemPool; class ChainstateManager; -class TxValidationState; struct ChainTxData; struct DisconnectedBlockTransactions; @@ -181,12 +182,46 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); /** Prune block files up to a given height */ void PruneBlockFilesManual(int nManualPruneHeight); -/** (try to) add transaction to memory pool - * plTxnReplaced will be appended to with all transactions replaced from mempool - * @param[out] fee_out optional argument to return tx fee to the caller **/ -bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, - std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, bool test_accept=false, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** +* Validation result for a single transaction mempool acceptance. +*/ +struct MempoolAcceptResult { + /** Used to indicate the results of mempool validation, + * including the possibility of unfinished validation. + */ + enum class ResultType { + VALID, //!> Fully validated, valid. + INVALID, //!> Invalid. + }; + ResultType m_result_type; + TxValidationState m_state; + + // The following fields are only present when m_result_type = ResultType::VALID + /** Mempool transactions replaced by the tx per BIP 125 rules. */ + std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Raw base fees. */ + std::optional<CAmount> m_base_fees; + + /** Constructor for failure case */ + explicit MempoolAcceptResult(TxValidationState state) + : m_result_type(ResultType::INVALID), + m_state(state), m_replaced_transactions(nullopt), m_base_fees(nullopt) { + Assume(!state.IsValid()); // Can be invalid or error + } + + /** Constructor for success case */ + explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) + : m_result_type(ResultType::VALID), m_state(TxValidationState{}), + m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} +}; + +/** + * (Try to) add a transaction to the memory pool. + * @param[in] bypass_limits When true, don't enforce mempool fee limits. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + */ +MempoolAcceptResult AcceptToMemoryPool(CTxMemPool& pool, const CTransactionRef& tx, + bool bypass_limits, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); @@ -802,7 +837,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_ibd_chainstate; + std::unique_ptr<CChainState> m_ibd_chainstate GUARDED_BY(::cs_main); //! A chainstate initialized on the basis of a UTXO snapshot. If this is //! non-null, it is always our active chainstate. @@ -815,7 +850,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_snapshot_chainstate; + std::unique_ptr<CChainState> m_snapshot_chainstate GUARDED_BY(::cs_main); //! Points to either the ibd or snapshot chainstate; indicates our //! most-work chain. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 8505ddc309..99803a91d2 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -934,9 +934,9 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TxoutType::NONSTANDARD: case TxoutType::WITNESS_UNKNOWN: case TxoutType::WITNESS_V1_TAPROOT: - default: return "unrecognized script"; - } + } // no default case, so the compiler can warn about missing cases + CHECK_NONFATAL(false); } static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 15972fe7bb..55110f30d9 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -94,8 +94,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s TxoutType whichType = Solver(scriptPubKey, vSolutions); CKeyID keyID; - switch (whichType) - { + switch (whichType) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -194,7 +193,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s } break; } - } + } // no default case, so the compiler can warn about missing cases if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { ret = std::max(ret, IsMineResult::WATCH_ONLY); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 27179839b7..b2eb8e4bca 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -30,8 +30,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - BOOST_CHECK(filename == test_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, test_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_directory) @@ -41,8 +41,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_directory) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename); - BOOST_CHECK(filename == expected_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, expected_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index e70b56c529..45e1b8c4b8 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -19,7 +19,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index bc90491a2c..b2cb0bf479 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -103,10 +103,8 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& name) +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) { - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); - if (args.IsArgSet("-format") && command != "createfromdump") { tfm::format(std::cerr, "The -format option can only be used with the \"createfromdump\" command.\n"); return false; @@ -119,6 +117,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (command == "create" && !args.IsArgSet("-wallet")) { + tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + return false; + } + const std::string name = args.GetArg("-wallet", ""); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); if (command == "create") { DatabaseOptions options; diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index f544a6f727..f4516bb5bc 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -10,7 +10,7 @@ namespace WalletTool { void WalletShowInfo(CWallet* wallet_instance); -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& file); +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command); } // namespace WalletTool diff --git a/test/functional/README.md b/test/functional/README.md index 2d04413eb2..d830ba0334 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -63,10 +63,13 @@ don't have test cases for. - Avoid stop-starting the nodes multiple times during the test if possible. A stop-start takes several seconds, so doing it several times blows up the runtime of the test. -- Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether - or not to use the cached data directories. The cached data directories - contain a 200-block pre-mined blockchain and wallets for four nodes. Each node - has 25 mature blocks (25x50=1250 BTC) in its wallet. +- Set the `self.setup_clean_chain` variable in `set_test_params()` to `True` to + initialize an empty blockchain and start from the Genesis block, rather than + load a premined blockchain from cache with the default value of `False`. The + cached data directories contain a 200-block pre-mined blockchain with the + spendable mining rewards being split between four nodes. Each node has 25 + mature block subsidies (25x50=1250 BTC) in its wallet. Using them is much more + efficient than mining blocks in your test. - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 6e72db1d96..fab921ef19 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -57,7 +57,7 @@ class BadTxTemplate: __metaclass__ = abc.ABCMeta # The expected error code given by bitcoind upon submission of the tx. - reject_reason = "" # type: Optional[str] + reject_reason: Optional[str] = "" # Only specified if it differs from mempool acceptance error. block_reject_reason = "" diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 97f24e1b6e..a0eb213a78 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -76,6 +76,9 @@ class ExampleTest(BitcoinTestFramework): """Override test parameters for your individual test. This method must be overridden and num_nodes must be explicitly set.""" + # By default every test loads a pre-mined chain of 200 blocks from cache. + # Set setup_clean_chain to True to skip this and start from the Genesis + # block. self.setup_clean_chain = True self.num_nodes = 3 # Use self.extra_args to change command-line arguments for the nodes diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 2c6553fbe2..5fcecb4882 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -36,7 +36,6 @@ def expected_messages(filename): class AsmapTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def test_without_asmap_arg(self): diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 603d7f5d3b..1a148f04f4 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -29,9 +29,11 @@ Start three nodes: block 200. node2 will reject block 102 since it's assumed valid, but it isn't buried by at least two weeks' work. """ -import time -from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, @@ -79,24 +81,6 @@ class AssumeValidTest(BitcoinTestFramework): assert not p2p_conn.is_connected break - def assert_blockchain_height(self, node, height): - """Wait until the blockchain is no longer advancing and verify it's reached the expected height.""" - last_height = node.getblock(node.getbestblockhash())['height'] - timeout = 10 - while True: - time.sleep(0.25) - current_height = node.getblock(node.getbestblockhash())['height'] - if current_height != last_height: - last_height = current_height - if timeout < 0: - assert False, "blockchain too short after timeout: %d" % current_height - timeout - 0.25 - continue - elif current_height > height: - assert False, "blockchain too long: %d" % current_height - elif current_height == height: - break - def run_test(self): p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) @@ -177,7 +161,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node0. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p0) - self.assert_blockchain_height(self.nodes[0], 101) + self.wait_until(lambda: self.nodes[0].getblockcount() >= 101) + assert_equal(self.nodes[0].getblockcount(), 101) # Send all blocks to node1. All blocks will be accepted. for i in range(2202): @@ -188,7 +173,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node2. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p2) - self.assert_blockchain_height(self.nodes[2], 101) + self.wait_until(lambda: self.nodes[2].getblockcount() >= 101) + assert_equal(self.nodes[2].getblockcount(), 101) if __name__ == '__main__': diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index b161c71a85..e6a53b52db 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -354,73 +354,75 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): hdkeypath = v17_info["hdkeypath"] pubkey = v17_info["pubkey"] - # Copy the 0.16 wallet to the last Bitcoin Core version and open it: - shutil.copyfile( - os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), - os.path.join(node_master_wallets_dir, "u1_v16") - ) - load_res = node_master.loadwallet("u1_v16") - # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 - assert_equal(load_res['warning'], '') - wallet = node_master.get_wallet_rpc("u1_v16") - info = wallet.getaddressinfo(v16_addr) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it - os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) - shutil.copyfile( - os.path.join(node_master_wallets_dir, "u1_v16"), - os.path.join(node_v16_wallets_dir, "wallets/u1_v16") - ) - self.start_node(-1, extra_args=["-wallet=u1_v16"]) - wallet = node_v16.get_wallet_rpc("u1_v16") - info = wallet.validateaddress(v16_addr) - assert_equal(info, v16_info) - - # Copy the 0.17 wallet to the last Bitcoin Core version and open it: - node_v17.unloadwallet("u1_v17") - shutil.copytree( - os.path.join(node_v17_wallets_dir, "u1_v17"), - os.path.join(node_master_wallets_dir, "u1_v17") - ) - node_master.loadwallet("u1_v17") - wallet = node_master.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it - node_master.unloadwallet("u1_v17") - shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "u1_v17"), - os.path.join(node_v17_wallets_dir, "u1_v17") - ) - node_v17.loadwallet("u1_v17") - wallet = node_v17.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - assert_equal(info, v17_info) - - # Copy the 0.19 wallet to the last Bitcoin Core version and open it: - shutil.copytree( - os.path.join(node_v19_wallets_dir, "w1_v19"), - os.path.join(node_master_wallets_dir, "w1_v19") - ) - node_master.loadwallet("w1_v19") - wallet = node_master.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + if self.is_bdb_compiled(): + # Old wallets are BDB and will only work if BDB is compiled + # Copy the 0.16 wallet to the last Bitcoin Core version and open it: + shutil.copyfile( + os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), + os.path.join(node_master_wallets_dir, "u1_v16") + ) + load_res = node_master.loadwallet("u1_v16") + # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 + assert_equal(load_res['warning'], '') + wallet = node_master.get_wallet_rpc("u1_v16") + info = wallet.getaddressinfo(v16_addr) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it + os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) + shutil.copyfile( + os.path.join(node_master_wallets_dir, "u1_v16"), + os.path.join(node_v16_wallets_dir, "wallets/u1_v16") + ) + self.start_node(-1, extra_args=["-wallet=u1_v16"]) + wallet = node_v16.get_wallet_rpc("u1_v16") + info = wallet.validateaddress(v16_addr) + assert_equal(info, v16_info) - # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it - node_master.unloadwallet("w1_v19") - shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "w1_v19"), - os.path.join(node_v19_wallets_dir, "w1_v19") - ) - node_v19.loadwallet("w1_v19") - wallet = node_v19.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + # Copy the 0.17 wallet to the last Bitcoin Core version and open it: + node_v17.unloadwallet("u1_v17") + shutil.copytree( + os.path.join(node_v17_wallets_dir, "u1_v17"), + os.path.join(node_master_wallets_dir, "u1_v17") + ) + node_master.loadwallet("u1_v17") + wallet = node_master.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: + shutil.copytree( + os.path.join(node_v19_wallets_dir, "w1_v19"), + os.path.join(node_master_wallets_dir, "w1_v19") + ) + node_master.loadwallet("w1_v19") + wallet = node_master.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index f9ece244fb..2b56bc78f5 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -49,7 +49,6 @@ from test_framework.util import ( class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 - self.setup_clean_chain = False self.rpc_timeout = 480 self.supports_cli = False diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 7de9a589be..2798d11b0a 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -4,6 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" import os +import random +import string from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch @@ -27,11 +29,21 @@ class FilelockTest(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): - self.nodes[0].createwallet(self.default_wallet_name) - wallet_dir = os.path.join(datadir, 'wallets') - self.log.info("Check that we can't start a second bitcoind instance using the same wallet") - expected_msg = "Error: Error initializing wallet database environment" - self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + self.default_wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + def check_wallet_filelock(descriptors): + wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)]) + self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors) + wallet_dir = os.path.join(datadir, 'wallets') + self.log.info("Check that we can't start a second bitcoind instance using the same wallet") + if descriptors: + expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + else: + expected_msg = "Error: Error initializing wallet database environment" + self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + + if self.is_bdb_compiled(): + check_wallet_filelock(False) + if self.is_sqlite_compiled(): + check_wallet_filelock(True) if __name__ == '__main__': FilelockTest().main() diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 6f1a0cd348..f22b7f266a 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -20,7 +20,6 @@ from test_framework.test_framework import BitcoinTestFramework class IncludeConfTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def setup_chain(self): diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index f2313bac13..b068ce612c 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -5,11 +5,11 @@ """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - hex_str_to_bytes, ) # Linux allow all characters other than \x00 @@ -49,6 +49,31 @@ class NotificationsTest(BitcoinTestFramework): super().setup_network() def run_test(self): + if self.is_wallet_compiled(): + # Setup the descriptors to be imported to the wallet + seed = "cTdGmKFWpbvpKQ7ejrdzqYT2hhjyb3GPHnLAK7wdi5Em67YLwSm9" + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + desc_imports = [{ + "desc": descsum_create("wpkh(" + xpriv + "/0/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + },{ + "desc": descsum_create("wpkh(" + xpriv + "/1/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + "internal": True, + }] + # Make the wallets and import the descriptors + # Ensures that node 0 and node 1 share the same wallet for the conflicting transaction tests below. + for i, name in enumerate(self.wallet_names): + self.nodes[i].createwallet(wallet_name=name, descriptors=self.options.descriptors, blank=True, load_on_startup=True) + if self.options.descriptors: + self.nodes[i].importdescriptors(desc_imports) + else: + self.nodes[i].sethdseed(True, seed) + self.log.info("test -blocknotify") block_count = 10 blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) @@ -84,11 +109,10 @@ class NotificationsTest(BitcoinTestFramework): for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) - # Conflicting transactions tests. Give node 0 same wallet seed as - # node 1, generate spends from node 0, and check notifications + # Conflicting transactions tests. + # Generate spends from node 0, and check notifications # triggered by node 1 self.log.info("test -walletnotify with conflicting transactions") - self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) self.nodes[0].rescanblockchain() self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) self.sync_blocks() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 6ee2b72c11..5027a9828f 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -517,7 +517,6 @@ def add_spender(spenders, *args, **kwargs): def random_checksig_style(pubkey): """Creates a random CHECKSIG* tapscript that would succeed with only the valid signature on witness stack.""" - return bytes(CScript([pubkey, OP_CHECKSIG])) opcode = random.choice([OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKSIGADD]) if (opcode == OP_CHECKSIGVERIFY): ret = CScript([pubkey, opcode, OP_1]) diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py new file mode 100755 index 0000000000..6e6046d84d --- /dev/null +++ b/test/functional/feature_utxo_set_hash.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test UTXO set hash value calculation in gettxoutsetinfo.""" + +import struct + +from test_framework.blocktools import create_transaction +from test_framework.messages import ( + CBlock, + COutPoint, + FromHex, +) +from test_framework.muhash import MuHash3072 +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class UTXOSetHashTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_deterministic_hash_results(self): + self.log.info("Test deterministic UTXO set hash results") + + # These depend on the setup_clean_chain option, the chain loaded from the cache + assert_equal(self.nodes[0].gettxoutsetinfo()['hash_serialized_2'], "b32ec1dda5a53cd025b95387aad344a801825fe46a60ff952ce26528f01d3be8") + assert_equal(self.nodes[0].gettxoutsetinfo("muhash")['muhash'], "dd5ad2a105c2d29495f577245c357409002329b9f4d6182c0af3dc2f462555c8") + + def test_muhash_implementation(self): + self.log.info("Test MuHash implementation consistency") + + node = self.nodes[0] + + # Generate 100 blocks and remove the first since we plan to spend its + # coinbase + block_hashes = node.generate(100) + blocks = list(map(lambda block: FromHex(CBlock(), node.getblock(block, False)), block_hashes)) + spending = blocks.pop(0) + + # Create a spending transaction and mine a block which includes it + tx = create_transaction(node, spending.vtx[0].rehash(), node.getnewaddress(), amount=49) + txid = node.sendrawtransaction(hexstring=tx.serialize_with_witness().hex(), maxfeerate=0) + + tx_block = node.generateblock(output=node.getnewaddress(), transactions=[txid]) + blocks.append(FromHex(CBlock(), node.getblock(tx_block['hash'], False))) + + # Serialize the outputs that should be in the UTXO set and add them to + # a MuHash object + muhash = MuHash3072() + + for height, block in enumerate(blocks): + # The Genesis block coinbase is not part of the UTXO set and we + # spent the first mined block + height += 2 + + for tx in block.vtx: + for n, tx_out in enumerate(tx.vout): + coinbase = 1 if not tx.vin[0].prevout.hash else 0 + + # Skip witness commitment + if (coinbase and n > 0): + continue + + data = COutPoint(int(tx.rehash(), 16), n).serialize() + data += struct.pack("<i", height * 2 + coinbase) + data += tx_out.serialize() + + muhash.insert(data) + + finalized = muhash.digest() + node_muhash = node.gettxoutsetinfo("muhash")['muhash'] + + assert_equal(finalized[::-1].hex(), node_muhash) + + def run_test(self): + self.test_deterministic_hash_results() + self.test_muhash_implementation() + + +if __name__ == '__main__': + UTXOSetHashTest().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 1257dff1ae..2cf0ef2251 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -29,6 +29,8 @@ class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + if self.is_wallet_compiled(): + self.requires_wallet = True def skip_test_if_missing_module(self): self.skip_if_no_cli() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 946bfa51d4..e9f61be4d4 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -62,6 +62,8 @@ class ZMQSubscriber: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + if self.is_wallet_compiled(): + self.requires_wallet = True def skip_test_if_missing_module(self): self.skip_if_no_py3_zmq() diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 70cd4ebb3b..752b925b92 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -69,6 +69,8 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + total_fee_old = self.nodes[0].getmempoolinfo()['total_fee'] + self.log.debug("Prioritize a transaction on node0") fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'], fees['modified']) @@ -76,6 +78,10 @@ class MempoolPersistTest(BitcoinTestFramework): fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.info('Check the total base fee is unchanged after prioritisetransaction') + assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee']) + assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items())) + tx_creation_time = self.nodes[0].getmempoolentry(txid=last_txid)['time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py index a63c3a3287..a04ba5db2d 100755 --- a/test/functional/p2p_add_connections.py +++ b/test/functional/p2p_add_connections.py @@ -17,7 +17,6 @@ def check_node_connections(*, node, num_in, num_out): class P2PAddConnections(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def setup_network(self): diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 91fbd722cf..69821763bd 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -46,7 +46,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index c592ab52b1..6584efae79 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -15,7 +15,6 @@ from test_framework.util import assert_equal class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [["-blocksonly"]] diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 642a217047..458e5235b6 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -81,7 +81,6 @@ class P2PBloomFilter(P2PInterface): class FilterTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [[ '-peerbloomfilters', diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index 2b75ad5175..d375af6fe1 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -41,7 +41,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index e4fc9fd178..f884cf90ff 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -13,7 +13,6 @@ from test_framework.test_framework import BitcoinTestFramework class InvalidLocatorTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = False def run_test(self): node = self.nodes[0] # convenience reference to the node diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py new file mode 100755 index 0000000000..080b2d93ad --- /dev/null +++ b/test/functional/p2p_message_capture.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test per-peer message capture capability. + +Additionally, the output of contrib/message-capture/message-capture-parser.py should be verified manually. +""" + +import glob +from io import BytesIO +import os + +from test_framework.p2p import P2PDataStore, MESSAGEMAP +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +def mini_parser(dat_file): + """Parse a data file created by CaptureMessage. + + From the data file we'll only check the structure. + + We won't care about things like: + - Deserializing the payload of the message + - This is managed by the deserialize methods in test_framework.messages + - The order of the messages + - There's no reason why we can't, say, change the order of the messages in the handshake + - Message Type + - We can add new message types + + We're ignoring these because they're simply too brittle to test here. + """ + with open(dat_file, 'rb') as f_in: + # This should have at least one message in it + assert(os.fstat(f_in.fileno()).st_size >= TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + while True: + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + tmp_header.read(TIME_SIZE) # skip the timestamp field + raw_msgtype = tmp_header.read(MSGTYPE_SIZE) + msgtype: bytes = raw_msgtype.split(b'\x00', 1)[0] + remainder = raw_msgtype.split(b'\x00', 1)[1] + assert(len(msgtype) > 0) + assert(msgtype in MESSAGEMAP) + assert(len(remainder) == 0 or not remainder.decode().isprintable()) + length: int = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") + data = f_in.read(length) + assert_equal(len(data), length) + + + +class MessageCaptureTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-capturemessages"]] + self.setup_clean_chain = True + + def run_test(self): + capturedir = os.path.join(self.nodes[0].datadir, "regtest/message_capture") + # Connect a node so that the handshake occurs + self.nodes[0].add_p2p_connection(P2PDataStore()) + self.nodes[0].disconnect_p2ps() + recv_file = glob.glob(os.path.join(capturedir, "*/msgs_recv.dat"))[0] + mini_parser(recv_file) + sent_file = glob.glob(os.path.join(capturedir, "*/msgs_sent.dat"))[0] + mini_parser(sent_file) + + +if __name__ == '__main__': + MessageCaptureTest().main() diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 8a751c6b54..4bf96cb0e6 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -56,7 +56,6 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def test_tx_requests(self): diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 99be6b7b8e..84ca1b99c2 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -268,6 +268,18 @@ class BlockchainTest(BitcoinTestFramework): res5 = node.gettxoutsetinfo(hash_type='none') assert 'hash_serialized_2' not in res5 + # hash_type muhash should return a different UTXO set hash. + res6 = node.gettxoutsetinfo(hash_type='muhash') + assert 'muhash' in res6 + assert(res['hash_serialized_2'] != res6['muhash']) + + # muhash should not be included in gettxoutset unless requested. + for r in [res, res2, res3, res4, res5]: + assert 'muhash' not in r + + # Unknown hash_type raises an error + assert_raises_rpc_error(-8, "foohash is not a valid hash_type", node.gettxoutsetinfo, "foohash") + def _test_getblockheader(self): node = self.nodes[0] diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 81862ac69e..51b7efb4c3 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -14,7 +14,6 @@ from test_framework.util import assert_raises_rpc_error class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index de0b7f303f..cf46616681 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -25,6 +25,7 @@ from test_framework.util import ( assert_raises_rpc_error, p2p_port, ) +from test_framework.wallet import MiniWallet def assert_net_servicesnames(servicesflag, servicenames): @@ -48,6 +49,9 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + # We need miniwallet to make a transaction + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.generate(1) # Get out of IBD for the minfeefilter and getpeerinfo tests. self.nodes[0].generate(101) @@ -74,8 +78,7 @@ class NetTest(BitcoinTestFramework): def test_getpeerinfo(self): self.log.info("Test getpeerinfo") # Create a few getpeerinfo last_block/last_transaction values. - if self.is_wallet_compiled(): - self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results self.nodes[1].generate(1) self.sync_all() time_now = int(time.time()) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b364077a9a..ed6abaed78 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -24,7 +24,6 @@ MAX_BIP125_RBF_SEQUENCE = 0xfffffffd class PSBTTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 3 self.extra_args = [ ["-walletrbf=1"], diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index e86f91b1d0..6177970872 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -10,6 +10,7 @@ Test corresponds to code in rpc/server.cpp. import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error class UptimeTest(BitcoinTestFramework): @@ -18,8 +19,12 @@ class UptimeTest(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): + self._test_negative_time() self._test_uptime() + def _test_negative_time(self): + assert_raises_rpc_error(-8, "Mocktime can not be negative: -1.", self.nodes[0].setmocktime, -1) + def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index f6ea9ef682..b8e899d675 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -178,7 +178,7 @@ can be called after the TestShell is shut down. | `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | | `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | | `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | -| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `setup_clean_chain` | `False` | A 200-block-long chain is initialized from cache by default. Instead, `setup_clean_chain` initializes an empty blockchain if set to `True`. | | `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | | `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | | `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py index 9de358aa0a..97b9c1d6d0 100644 --- a/test/functional/test_framework/bdb.py +++ b/test/functional/test_framework/bdb.py @@ -51,7 +51,6 @@ def dump_leaf_page(data): page_info['pgno'] = pgno page_info['prev_pgno'] = prev_pgno page_info['next_pgno'] = next_pgno - page_info['entries'] = entries page_info['hf_offset'] = hf_offset page_info['level'] = level page_info['pg_type'] = pg_type diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 6ad4e13db2..27a09ef86c 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1273,7 +1273,7 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire # note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("msgtype", "data") + __slots__ = ("data") def __init__(self, msgtype, data=None): self.msgtype = msgtype diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index be0e9f24e2..c35533698c 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -29,8 +29,6 @@ MAX_SCRIPT_ELEMENT_SIZE = 520 LOCKTIME_THRESHOLD = 500000000 ANNEX_TAG = 0x50 -OPCODE_NAMES = {} # type: Dict[CScriptOp, str] - LEAF_VERSION_TAPSCRIPT = 0xc0 def hash160(s): @@ -47,7 +45,6 @@ def bn2vch(v): # Serialize to bytes return encoded_v.to_bytes(n_bytes, 'little') -_opcode_instances = [] # type: List[CScriptOp] class CScriptOp(int): """A single script opcode""" __slots__ = () @@ -111,6 +108,9 @@ class CScriptOp(int): _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] +OPCODE_NAMES: Dict[CScriptOp, str] = {} +_opcode_instances: List[CScriptOp] = [] + # Populate opcode instance table for n in range(0xff + 1): CScriptOp(n) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 4bda73599d..70a9798449 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -108,6 +108,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # skipped. If list is truncated, wallet creation is skipped and keys # are not imported. self.wallet_names = None + # By default the wallet is not required. Set to true by skip_if_no_wallet(). + # When False, we ignore wallet_names regardless of what it is. + self.requires_wallet = False self.set_test_params() assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes if self.options.timeout_factor == 0 : @@ -184,15 +187,30 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') group = parser.add_mutually_exclusive_group() - group.add_argument("--descriptors", default=False, action="store_true", + group.add_argument("--descriptors", action='store_const', const=True, help="Run test using a descriptor wallet", dest='descriptors') - group.add_argument("--legacy-wallet", default=False, action="store_false", + group.add_argument("--legacy-wallet", action='store_const', const=False, help="Run test using legacy wallets", dest='descriptors') self.add_options(parser) self.options = parser.parse_args() self.options.previous_releases_path = previous_releases_path + config = configparser.ConfigParser() + config.read_file(open(self.options.configfile)) + self.config = config + + if self.options.descriptors is None: + # Prefer BDB unless it isn't available + if self.is_bdb_compiled(): + self.options.descriptors = False + elif self.is_sqlite_compiled(): + self.options.descriptors = True + else: + # If neither are compiled, tests requiring a wallet will be skipped and the value of self.options.descriptors won't matter + # It still needs to exist and be None in order for tests to work however. + self.options.descriptors = None + def setup(self): """Call this method to start up the test framework object with options set.""" @@ -202,9 +220,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.cachedir = os.path.abspath(self.options.cachedir) - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - self.config = config + config = self.config + fname_bitcoind = os.path.join( config["environment"]["BUILDDIR"], "src", @@ -377,7 +394,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() - if self.is_wallet_compiled(): + if self.requires_wallet: self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: @@ -769,10 +786,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def skip_if_no_wallet(self): """Skip the running test if wallet has not been compiled.""" + self.requires_wallet = True if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") if self.options.descriptors: self.skip_if_no_sqlite() + else: + self.skip_if_no_bdb() def skip_if_no_sqlite(self): """Skip the running test if sqlite has not been compiled.""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index c652ac0a06..354cacf95a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -86,29 +86,29 @@ EXTENDED_SCRIPTS = [ BASE_SCRIPTS = [ # Scripts that are run by default. # Longest test should go first, to favor running tests in parallel - 'wallet_hd.py', + 'wallet_hd.py --legacy-wallet', 'wallet_hd.py --descriptors', - 'wallet_backup.py', + 'wallet_backup.py --legacy-wallet', 'wallet_backup.py --descriptors', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', 'feature_maxuploadtarget.py', 'feature_block.py', - 'rpc_fundrawtransaction.py', + 'rpc_fundrawtransaction.py --legacy-wallet', 'rpc_fundrawtransaction.py --descriptors', 'p2p_compactblocks.py', 'feature_segwit.py --legacy-wallet', # vv Tests less than 2m vv - 'wallet_basic.py', + 'wallet_basic.py --legacy-wallet', 'wallet_basic.py --descriptors', - 'wallet_labels.py', + 'wallet_labels.py --legacy-wallet', 'wallet_labels.py --descriptors', 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'wallet_listtransactions.py', + 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', 'feature_taproot.py', # vv Tests less than 60s vv @@ -116,21 +116,21 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', - 'wallet_listreceivedby.py', + 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', - 'wallet_abandonconflict.py', + 'wallet_abandonconflict.py --legacy-wallet', 'wallet_abandonconflict.py --descriptors', 'feature_csv_activation.py', - 'rpc_rawtransaction.py', + 'rpc_rawtransaction.py --legacy-wallet', 'rpc_rawtransaction.py --descriptors', - 'wallet_address_types.py', + 'wallet_address_types.py --legacy-wallet', 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', 'p2p_feefilter.py', 'feature_reindex.py', 'feature_abortnode.py', # vv Tests less than 30s vv - 'wallet_keypool_topup.py', + 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', @@ -138,7 +138,7 @@ BASE_SCRIPTS = [ 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', - 'tool_wallet.py', + 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', @@ -146,14 +146,14 @@ BASE_SCRIPTS = [ 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', - 'wallet_avoidreuse.py', + 'wallet_avoidreuse.py --legacy-wallet', 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', - 'wallet_multiwallet.py', + 'wallet_multiwallet.py --legacy-wallet', 'wallet_multiwallet.py --descriptors', 'wallet_multiwallet.py --usecli', - 'wallet_createwallet.py', + 'wallet_createwallet.py --legacy-wallet', 'wallet_createwallet.py --usecli', 'wallet_createwallet.py --descriptors', 'wallet_watchonly.py --legacy-wallet', @@ -161,27 +161,27 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', - 'rpc_psbt.py', + 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', - 'rpc_signrawtransaction.py', + 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', - 'wallet_groups.py', + 'wallet_groups.py --legacy-wallet', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', 'p2p_disconnect_ban.py', 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py', + 'wallet_disable.py --legacy-wallet', 'wallet_disable.py --descriptors', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', 'rpc_net.py', - 'wallet_keypool.py', + 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', 'p2p_nobloomfilter_messages.py', @@ -195,70 +195,72 @@ BASE_SCRIPTS = [ 'p2p_invalid_tx.py', 'feature_assumevalid.py', 'example_test.py', - 'wallet_txn_doublespend.py', + 'wallet_txn_doublespend.py --legacy-wallet', 'wallet_txn_doublespend.py --descriptors', - 'feature_backwards_compatibility.py', + 'feature_backwards_compatibility.py --legacy-wallet', 'feature_backwards_compatibility.py --descriptors', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', + 'feature_utxo_set_hash.py', 'feature_rbf.py', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py', + 'rpc_createmultisig.py --legacy-wallet', 'rpc_createmultisig.py --descriptors', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', - 'wallet_importprunedfunds.py', + 'wallet_importprunedfunds.py --legacy-wallet', 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', 'rpc_signmessage.py', 'rpc_generateblock.py', 'rpc_generate.py', - 'wallet_balance.py', + 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', - 'feature_nulldummy.py', + 'feature_nulldummy.py --legacy-wallet', 'feature_nulldummy.py --descriptors', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py --legacy-wallet', 'wallet_import_with_label.py --legacy-wallet', 'wallet_importdescriptors.py --descriptors', - 'wallet_upgradewallet.py', + 'wallet_upgradewallet.py --legacy-wallet', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', 'mining_basic.py', 'feature_signet.py', - 'wallet_bumpfee.py', + 'wallet_bumpfee.py --legacy-wallet', 'wallet_bumpfee.py --descriptors', 'wallet_implicitsegwit.py --legacy-wallet', 'rpc_named_arguments.py', - 'wallet_listsinceblock.py', + 'wallet_listsinceblock.py --legacy-wallet', 'wallet_listsinceblock.py --descriptors', 'wallet_listdescriptors.py --descriptors', 'p2p_leak.py', - 'wallet_encryption.py', + 'wallet_encryption.py --legacy-wallet', 'wallet_encryption.py --descriptors', 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', - 'wallet_resendwallettransactions.py', + 'wallet_resendwallettransactions.py --legacy-wallet', 'wallet_resendwallettransactions.py --descriptors', - 'wallet_fallbackfee.py', + 'wallet_fallbackfee.py --legacy-wallet', 'wallet_fallbackfee.py --descriptors', 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', - 'wallet_create_tx.py', - 'wallet_send.py', + 'wallet_create_tx.py --legacy-wallet', + 'wallet_send.py --legacy-wallet', + 'wallet_send.py --descriptors', 'wallet_create_tx.py --descriptors', 'p2p_fingerprint.py', 'feature_uacomment.py', - 'wallet_coinbase_category.py', + 'wallet_coinbase_category.py --legacy-wallet', 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', @@ -266,6 +268,7 @@ BASE_SCRIPTS = [ 'p2p_add_connections.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', + 'p2p_message_capture.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 8a1af24dcf..28103793df 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -183,11 +183,13 @@ class ToolWalletTest(BitcoinTestFramework): def test_invalid_tool_commands_and_args(self): self.log.info('Testing that various invalid commands raise with specific error messages') - self.assert_raises_tool_error('Invalid command: foo', 'foo') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'foo'", 'foo') # `bitcoin-wallet help` raises an error. Use `bitcoin-wallet -help`. - self.assert_raises_tool_error('Invalid command: help', 'help') - self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'help'", 'help') + self.assert_raises_tool_error('Error: Additional arguments provided (create). Methods do not take arguments. Please refer to `-help`.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('No method provided. Run `bitcoin-wallet -help` for valid methods.') + self.assert_raises_tool_error('Wallet name must be provided when creating a new wallet.', 'create') locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: @@ -348,7 +350,8 @@ class ToolWalletTest(BitcoinTestFramework): self.log.info('Checking createfromdump') self.do_tool_createfromdump("load", "wallet.dump") - self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") + if self.is_bdb_compiled(): + self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") if self.is_sqlite_compiled(): self.do_tool_createfromdump("load-sqlite", "wallet.dump", "sqlite") diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 229c134a4b..bc4fa90e83 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -65,7 +65,6 @@ def assert_balances(node, mine): class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index cf3317121f..16a0a50b07 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -17,7 +17,6 @@ from test_framework.wallet_util import bytes_to_wif, generate_wif_key class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 1de41a5f96..c04a15a67c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -23,11 +23,14 @@ class WalletDescriptorTest(BitcoinTestFramework): self.skip_if_no_sqlite() def run_test(self): - # Make a legacy wallet and check it is BDB - self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) - wallet_info = self.nodes[0].getwalletinfo() - assert_equal(wallet_info['format'], 'bdb') - self.nodes[0].unloadwallet("legacy1") + if self.is_bdb_compiled(): + # Make a legacy wallet and check it is BDB + self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) + wallet_info = self.nodes[0].getwalletinfo() + assert_equal(wallet_info['format'], 'bdb') + self.nodes[0].unloadwallet("legacy1") + else: + self.log.warning("Skipping BDB test") # Make a descriptor wallet self.log.info("Making a descriptor wallet") diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 9835c5a2af..880341fdd9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -8,6 +8,7 @@ from decimal import Decimal, getcontext from itertools import product from test_framework.authproxy import JSONRPCException +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -168,49 +169,91 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name="w1") w1 = self.nodes[1].get_wallet_rpc("w1") # w2 contains the private keys for w3 - self.nodes[1].createwallet(wallet_name="w2") + self.nodes[1].createwallet(wallet_name="w2", blank=True) w2 = self.nodes[1].get_wallet_rpc("w2") + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + xpub = "tpubD6NzVbkrYhZ4YkEfMbRJkQyZe7wTkbTNRECozCtJPtdLRn6cT1QKb8yHjwAPcAr26eHBFYs5iLiFFnCbwPRsncCKUKCfubHDMGKzMVcN1Jg" + if self.options.descriptors: + w2.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpriv + "/0/0/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True + },{ + "desc": descsum_create("wpkh(" + xpriv + "/0/1/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True, + "internal": True + }]) + else: + w2.sethdseed(True) + # w3 is a watch-only wallet, based on w2 self.nodes[1].createwallet(wallet_name="w3", disable_private_keys=True) w3 = self.nodes[1].get_wallet_rpc("w3") - for _ in range(3): - a2_receive = w2.getnewaddress() - a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation - res = w3.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], + if self.options.descriptors: + # Match the privkeys in w2 for descriptors + res = w3.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpub + "/0/0/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "watchonly": True },{ - "desc": w2.getaddressinfo(a2_change)["desc"], + "desc": descsum_create("wpkh(" + xpub + "/0/1/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "internal": True, "watchonly": True }]) assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w3 - self.nodes[0].generate(1) - self.sync_blocks() - - # w4 has private keys enabled, but only contains watch-only keys (from w2) - self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) - w4 = self.nodes[1].get_wallet_rpc("w4") for _ in range(3): a2_receive = w2.getnewaddress() - res = w4.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], - "timestamp": "now", - "keypool": False, - "watchonly": True - }]) - assert_equal(res, [{"success": True}]) + if not self.options.descriptors: + # Because legacy wallets use exclusively hardened derivation, we can't do a ranged import like we do for descriptors + a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation + res = w3.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": True, + "watchonly": True + },{ + "desc": w2.getaddressinfo(a2_change)["desc"], + "timestamp": "now", + "keypool": True, + "internal": True, + "watchonly": True + }]) + assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w4 + w0.sendtoaddress(a2_receive, 10) # fund w3 self.nodes[0].generate(1) self.sync_blocks() + if not self.options.descriptors: + # w4 has private keys enabled, but only contains watch-only keys (from w2) + # This is legacy wallet behavior only as descriptor wallets don't allow watchonly and non-watchonly things in the same wallet. + self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) + w4 = self.nodes[1].get_wallet_rpc("w4") + for _ in range(3): + a2_receive = w2.getnewaddress() + res = w4.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": False, + "watchonly": True + }]) + assert_equal(res, [{"success": True}]) + + w0.sendtoaddress(a2_receive, 10) # fund w4 + self.nodes[0].generate(1) + self.sync_blocks() + self.log.info("Send to address...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=True) @@ -241,11 +284,15 @@ class WalletSendTest(BitcoinTestFramework): res = w2.walletprocesspsbt(res["psbt"]) assert res["complete"] - self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") - self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) - res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) - res = w2.walletprocesspsbt(res["psbt"]) - assert res["complete"] + if not self.options.descriptors: + # Descriptor wallets do not allow mixed watch-only and non-watch-only things in the same wallet. + # This is specifically testing that w4 ignores its own private keys and creates a psbt with send + # which is not something that needs to be tested in descriptor wallets. + self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") + self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) + res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] self.log.info("Create OP_RETURN...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index d0bb6135a8..fbc0f995d2 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -57,6 +57,7 @@ class UpgradeWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_bdb() self.skip_if_no_previous_releases() def setup_network(self): diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index b0c41b2738..24799fe5f2 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -14,7 +14,6 @@ from test_framework.util import ( class CreateWalletWatchonlyTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index aa0aa11d15..611061072f 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -225,6 +225,8 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir) args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), '-merge=1', + '-shuffle=0', + '-prefer_small=1', '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), os.path.join(merge_dir, t), diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index a3c8f33e79..bf7aeb5b4f 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -60,8 +60,6 @@ EXPECTED_BOOST_INCLUDES=( boost/multi_index/ordered_index.hpp boost/multi_index/sequenced_index.hpp boost/multi_index_container.hpp - boost/preprocessor/cat.hpp - boost/preprocessor/stringize.hpp boost/process.hpp boost/signals2/connection.hpp boost/signals2/optional_last_value.hpp |