diff options
258 files changed, 4233 insertions, 2494 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b915cfabc7..691582239e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -82,10 +82,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [bionic]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_win64.sh" @@ -136,7 +136,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[no depends, sanitizers: fuzzer,address,undefined] [focal]' + name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -160,10 +160,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" task: - name: 'macOS 10.14 [gui, no tests] [bionic]' + name: 'macOS 10.14 [gui, no tests] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_mac.sh" diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4967e675f6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# This is the top-most EditorConfig file. +root = true + +# For all files. +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +# Source code files +[*.{h,cpp,py,sh}] +indent_size = 4 + +# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +[*.yml] +indent_size = 2 + +# Makefiles +[{*.am,Makefile.*.include}] +indent_style = tab + +# Autoconf scripts +[configure.ac] +indent_size = 2 diff --git a/Makefile.am b/Makefile.am index c554f630ae..f6b824faaa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,6 +46,9 @@ OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed DIST_CONTRIB = \ + $(top_srcdir)/test/sanitizer_suppressions/lsan \ + $(top_srcdir)/test/sanitizer_suppressions/tsan \ + $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ $(top_srcdir)/contrib/linearize/linearize-hashes.py @@ -141,7 +144,7 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) - $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) $(OSX_DMG): $(OSX_TEMP_ISO) $(DMG) dmg "$<" "$@" diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 75c43f9a92..5201a8cc7c 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -14,6 +14,9 @@ m4_define([_CHECK_ATOMIC_testbody], [[ #include <cstdint> int main() { + std::atomic<bool> lock{true}; + std::atomic_exchange(&lock, false); + std::atomic<int64_t> a{}; int64_t v = 5; diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 6a3c9f1dc1..490ce8b1ce 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -104,6 +104,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondesc.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondescdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionfilterproxy.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionoverviewwidget.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionrecord.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiontablemodel.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionview.cpp" /> diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index de89a09d6a..e38cfe8eef 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -21,7 +21,6 @@ test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/check-rpc-mappings.py . test/lint/lint-all.sh if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index b0de2ec0bb..6da011d19b 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,9 +7,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index a32de4a6b5..b7157c608d 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,5 +14,5 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++ --with-boost-process" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 567145fe47..bbc43cfabd 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -16,4 +16,5 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index affaaaa1aa..3eea0a4565 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) export HOST=x86_64-w64-mingw32 -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file" +export DPKG_ADD_ARCH="i386" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false -export RUN_SECURITY_TESTS="true" export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-fuzz-binary --disable-gui-tests --without-boost-process" # Compiler for MinGW-w64 causes false -Wreturn-type warning. # See https://sourceforge.net/p/mingw-w64/bugs/306/ diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 58a8983e6e..82964897e1 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -13,7 +13,7 @@ for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul, echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" - echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b" + echo "( wine \"${b}_orig\" \"\$@\" ) || ( sleep 1 && wine \"${b}_orig\" \"\$@\" )" >> "$b" chmod +x "$b" fi done diff --git a/configure.ac b/configure.ac index b64c22450c..4ac985abdc 100644 --- a/configure.ac +++ b/configure.ac @@ -132,6 +132,12 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) +AC_ARG_ENABLE([ebpf], + [AS_HELP_STRING([--enable-ebpf], + [enable eBPF tracing (default is yes if sys/sdt.h is found)])], + [use_ebpf=$enableval], + [use_ebpf=yes]) + AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], [enable UPNP (default is yes if libminiupnpc is found)])], @@ -178,10 +184,16 @@ AC_ARG_ENABLE([extended-functional-tests], AC_ARG_ENABLE([fuzz], AS_HELP_STRING([--enable-fuzz], - [enable building of fuzz targets (default no). enabling this will disable all other targets]), + [build for fuzzing (default no). enabling this will disable all other targets and override --{enable,disable}-fuzz-binary]), [enable_fuzz=$enableval], [enable_fuzz=no]) +AC_ARG_ENABLE([fuzz-binary], + AS_HELP_STRING([--enable-fuzz-binary], + [enable building of fuzz binary (default yes).]), + [enable_fuzz_binary=$enableval], + [enable_fuzz_binary=yes]) + AC_ARG_ENABLE([danger_fuzz_link_all], AS_HELP_STRING([--enable-danger-fuzz-link-all], [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]), @@ -421,6 +433,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -807,6 +820,11 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) + case $host in + powerpc64* | ppc64*) + AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) + ;; + esac else AC_SEARCH_LIBS([clock_gettime],[rt]) fi @@ -842,7 +860,10 @@ if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - AX_CHECK_COMPILE_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) + dnl -fcf-protection used with Clang 7 causes ld to emit warnings: + dnl ld: error: ... <corrupt x86 feature size: 0x8> + dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we wont use the flag in this case. + AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) dnl stack-clash-protection does not work properly when building for Windows. dnl We use the test case from https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 @@ -1212,7 +1233,7 @@ AC_DEFUN([SUPPRESS_WARNINGS], dnl enable-fuzz should disable all other targets if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets) + AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no @@ -1228,10 +1249,11 @@ if test "x$enable_fuzz" = "xyes"; then use_upnp=no use_natpmp=no use_zmq=no + enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AC_MSG_CHECKING([whether main function is needed]) + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], [AC_MSG_RESULT([no])], @@ -1242,8 +1264,10 @@ if test "x$enable_fuzz" = "xyes"; then #include <cstdint> #include <cstddef> extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* unterminated comment to remove the main function ... - ]],[[]])]) + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) else BITCOIN_QT_INIT @@ -1257,6 +1281,8 @@ else QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi + + CPPFLAGS="$CPPFLAGS -DPROVIDE_MAIN_FUNCTION" fi if test x$enable_wallet != xno; then @@ -1295,6 +1321,16 @@ if test x$enable_wallet != xno; then fi fi +if test x$use_ebpf != xno; then + AC_CHECK_HEADER([sys/sdt.h], [have_sdt=yes], [have_sdt=no]) +else + have_sdt=no +fi + +if test x$have_sdt = xyes; then + AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints]) +fi + dnl Check for libminiupnpc (optional) if test x$use_upnp != xno; then AC_CHECK_HEADERS( @@ -1689,8 +1725,10 @@ AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) +AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) @@ -1707,6 +1745,8 @@ AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) +AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) +AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) @@ -1855,6 +1895,7 @@ echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " with natpmp = $use_natpmp" echo " use asm = $use_asm" +echo " ebpf tracing = $have_sdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index f105968515..60acb0d593 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -13,15 +13,16 @@ def setup(): programs = ['ruby', 'git', 'make', 'wget', 'curl'] if args.kvm: programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) + elif args.docker: + if not os.path.isfile('/lib/systemd/system/docker.service'): + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install Docker.', file=sys.stderr) + sys.exit(1) else: programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) @@ -34,14 +35,14 @@ def setup(): if not os.path.isdir('bitcoin'): subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] elif not args.kvm: - make_image_prog += ['--lxc'] + make_image_prog += ['--lxc', '--disksize', '13000'] subprocess.check_call(make_image_prog) os.chdir(workdir) - if args.is_bionic and not args.kvm and not args.docker: + if args.is_focal and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') sys.exit(0) @@ -175,7 +176,7 @@ def main(): args = parser.parse_args() workdir = os.getcwd() - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 0e2c5be0fa..b06fc782a3 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-linux-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -15,6 +15,8 @@ packages: - "ca-certificates" - "curl" - "faketime" +- "g++-8" +- "gcc-8" - "git" - "libtool" - "patch" @@ -27,6 +29,12 @@ packages: # - aarch64-linux-gnu - "binutils-aarch64-linux-gnu" - "g++-8-aarch64-linux-gnu" +# - powerpc64-linux-gnu +- "binutils-powerpc64-linux-gnu" +- "g++-8-powerpc64-linux-gnu" +# - powerpc64le-linux-gnu +- "binutils-powerpc64le-linux-gnu" +- "g++-8-powerpc64le-linux-gnu" # - riscv64-linux-gnu - "binutils-riscv64-linux-gnu" - "g++-8-riscv64-linux-gnu" @@ -38,8 +46,8 @@ script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" @@ -64,7 +72,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -78,7 +86,13 @@ script: | echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + if [ "${i:0:11}" = "powerpc64le" ]; then + echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + elif [ "${i:0:9}" = "powerpc64" ]; then + echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + else + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} + fi chmod +x ${WRAP_DIR}/${i}-${prog} fi done @@ -95,7 +109,7 @@ script: | BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 done # Faketime for binaries @@ -118,7 +132,7 @@ script: | # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "riscv64-linux-gnu" ]; then + if [ "${i}" = "powerpc64-linux-gnu" ]; then # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 # TODO: remove this when no longer needed HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" @@ -132,7 +146,7 @@ script: | tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 2330ff7736..3f0c0c3332 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -2,15 +2,19 @@ name: "bitcoin-dmg-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: - "faketime" - "xorriso" +- "python3-pip" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" +- "url": "https://github.com/achow101/signapple.git" + "dir": "signapple" + "commit": "c7e73aa27a7615ac9506559173f787e2906b25eb" files: - "bitcoin-osx-unsigned.tar.gz" script: | @@ -27,15 +31,23 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done - UNSIGNED=bitcoin-osx-unsigned.tar.gz + # Install signapple + cd signapple + python3 -m pip install -U pip setuptools + python3 -m pip install . + export PATH="$HOME/.local/bin":$PATH + cd .. + + UNSIGNED_TARBALL=bitcoin-osx-unsigned.tar.gz + UNSIGNED_APP=dist/Bitcoin-Qt.app SIGNED=bitcoin-osx-signed.dmg - tar -xf ${UNSIGNED} + tar -xf ${UNSIGNED_TARBALL} OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED} signature/osx + ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 9a7dd13c9c..0dc531df0e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-osx-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -29,6 +29,7 @@ packages: - "python3-setuptools" - "fonts-tuffy" - "xorriso" +- "libtinfo5" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -39,7 +40,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg xorrisofs" @@ -63,7 +64,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -75,7 +76,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done @@ -138,8 +139,6 @@ script: | cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate - cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} pushd unsigned-app-${i} find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 6bcd126662..c13c24c3cc 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -2,7 +2,7 @@ name: "bitcoin-win-signer" distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index fc79745e69..95cf0185e2 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -3,7 +3,7 @@ name: "bitcoin-core-win-22" enable_cache: true distro: "ubuntu" suites: -- "bionic" +- "focal" architectures: - "amd64" packages: @@ -31,7 +31,7 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g -fno-ident" @@ -55,7 +55,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -67,7 +67,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done @@ -81,7 +81,7 @@ script: | echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 7e28d27454..db28cd07a0 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -1,41 +1,51 @@ 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) -617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa -E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach -152812300785C96444D3334D17565732E08E5E41 Andrew Chow +617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) +E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) +152812300785C96444D3334D17565732E08E5E41 Andrew Chow (achow101) 590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) -912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak +912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) -F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur -C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields -BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random -6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark +18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) +101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) +F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) +C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) +BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) +6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) -D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece -01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen +D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) +6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) +01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) -D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 -82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack -32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli -4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon +A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) +7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) +D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) +82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack (jonatack) +32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) +4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) -E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr -B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke +30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) +E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) 07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) -CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider -E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford +CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider (meshcollider) +E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford (fanquake) AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) -9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo -77E72E69DA7EE0A148C06B21B34821D4944DE5F7 Nils Schneider -F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger -D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy -37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd -D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) -133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo (michagogo) +C57E4B42223FDE851D4F69DD28DF2724F241D8EE midnightmagic (midnightmagic) +F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger (guggero, Oliver Gugger) +D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy (prab) +37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd (petertodd) +D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille [Location: Leuven, Belgium] (sipa) +133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille (sipa) 6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) -ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost +ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) +867345026B6763E8B07EE73AB6737117397F5C4F Stephan Oeste (Emzy) 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) -AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami -79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko -71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan +6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) +AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) +71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index dbe1ea837b..5870deee39 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -105,7 +105,8 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum Override the space-separated list of platform triples for which to perform a bootstrappable build. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf - aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32")_ + aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32 + x86_64-apple-darwin18")_ * _**SOURCES_PATH**_ diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 54cc5793f6..da6bd13f6a 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -65,12 +65,13 @@ else fi ################ -# Check 4: Make sure that build directories do no exist +# Check 4: Make sure that build directories do not exist ################ # Default to building for all supported HOSTs (overridable by environment) export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu - x86_64-w64-mingw32}" + x86_64-w64-mingw32 + x86_64-apple-darwin18}" DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" @@ -105,9 +106,28 @@ for host in $hosts_distsrc_exists; do done exit 1 else + mkdir -p "$DISTSRC_BASE" fi +################ +# Check 5: When building for darwin, make sure that the macOS SDK exists +################ + +for host in $HOSTS; do + case "$host" in + *darwin*) + OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')" + if [ -e "$OSX_SDK" ]; then + echo "Found macOS SDK at '${OSX_SDK}', using..." + else + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + exit 1 + fi + ;; + esac +done + ######### # Setup # ######### diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index b00c42ce01..8ea9e00596 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -38,7 +38,28 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Set environment variables to point Guix's cross-toolchain to the right + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export CPATH="${NATIVE_GCC}/include" +case "$HOST" in + *darwin*) + # When targeting darwin, some native tools built by depends require + # native packages not incorporated in depends + # + # libcap required by native_cdrkit/wodim + # zlib, bzip2 required by native_cdrkit/genisoimage + for native_pkg in libcap zlib bzip2; do + native_pkg_store_path=$(store_path "$native_pkg") + export LIBRARY_PATH="${native_pkg_store_path}/lib:${LIBRARY_PATH}" + export CPATH="${native_pkg_store_path}/include:${CPATH}" + done + ;; +esac + +# Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in *mingw*) @@ -48,14 +69,18 @@ case "$HOST" in CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" - export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" - export CPATH="${NATIVE_GCC}/include" - + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; *linux*) CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" @@ -64,9 +89,7 @@ case "$HOST" in CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because - # the limits.h in it is missing a '#include_next <limits.h>' - export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" ;; @@ -77,14 +100,25 @@ esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" for p in "${PATHS[@]}"; do - if [ ! -d "$p" ]; then + if [ -n "$p" ] && [ ! -d "$p" ]; then echo "'$p' doesn't exist or isn't a directory... Aborting..." exit 1 fi done # Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes +case "$HOST" in + *darwin*) + # The auto-rpath behavior is necessary for darwin builds as some native + # tools built by depends refer to and depend on Guix-built native + # libraries + # + # After the native packages in depends are built, the ld wrapper should + # no longer affect our build, as clang would instead reach for + # x86_64-apple-darwin18-ld from cctools + ;; + *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; +esac # Make /usr/bin if it doesn't exist [ -e /usr/bin ] || mkdir -p /usr/bin @@ -114,6 +148,16 @@ export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" export TZ="UTC" +case "$HOST" in + *darwin*) + # cctools AR, unlike GNU binutils AR, does not have a deterministic mode + # or a configure flag to enable determinism by default, it only + # understands if this env-var is set or not. See: + # + # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 + export ZERO_AR_DATE=yes + ;; +esac #################### # Depends Building # @@ -135,7 +179,8 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' + qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + FORCE_USE_SYSTEM_CLANG=1 ########################### @@ -159,7 +204,7 @@ fi ########################### # CONFIGFLAGS -CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" case "$HOST" in *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; esac @@ -169,6 +214,7 @@ HOST_CFLAGS="-O2 -g" case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; + *darwin*) unset HOST_CFLAGS ;; esac # CXXFLAGS @@ -199,8 +245,8 @@ mkdir -p "$DISTSRC" --disable-maintainer-mode \ --disable-dependency-tracking \ ${CONFIGFLAGS} \ - CFLAGS="${HOST_CFLAGS}" \ - CXXFLAGS="${HOST_CXXFLAGS}" \ + ${HOST_CFLAGS:+CFLAGS="${HOST_CFLAGS}"} \ + ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -210,14 +256,9 @@ mkdir -p "$DISTSRC" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - - case "$HOST" in - *linux*|*mingw*) - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} - ;; - esac + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} # Make the os-specific installers case "$HOST" in @@ -232,8 +273,39 @@ mkdir -p "$DISTSRC" INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH - make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + case "$HOST" in + *darwin*) + make install-strip DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + *) + make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + ;; + esac + case "$HOST" in + *darwin*) + make osx_volname ${V:+V=1} + make deploydir ${V:+V=1} + mkdir -p "unsigned-app-${HOST}" + cp --target-directory="unsigned-app-${HOST}" \ + osx_volname \ + contrib/macdeploy/detached-sig-{apply,create}.sh \ + "${BASEPREFIX}/${HOST}"/native/bin/dmg + for util in codesign_allocate pagestuff; do + cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util" + done + mv --target-directory="unsigned-app-${HOST}" dist + ( + cd "unsigned-app-${HOST}" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + ) + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + ;; + esac ( cd installed @@ -248,13 +320,18 @@ mkdir -p "$DISTSRC" find . -name "lib*.a" -delete # Prune pkg-config files - rm -r "${DISTNAME}/lib/pkgconfig" + rm -rf "${DISTNAME}/lib/pkgconfig" - # Split binaries and libraries from their debug symbols - { - find "${DISTNAME}/bin" -type f -executable -print0 - find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + case "$HOST" in + *darwin*) ;; + *) + # Split binaries and libraries from their debug symbols + { + find "${DISTNAME}/bin" -type f -executable -print0 + find "${DISTNAME}/lib" -type f -print0 + } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + ;; + esac case "$HOST" in *mingw*) @@ -294,6 +371,13 @@ mkdir -p "$DISTSRC" | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) ;; + *darwin*) + find "${DISTNAME}" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) + ;; esac ) ) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 5e011ea184..3b89659263 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -3,28 +3,44 @@ (gnu packages autotools) (gnu packages base) (gnu packages bash) + (gnu packages cdrom) (gnu packages check) + (gnu packages cmake) (gnu packages commencement) (gnu packages compression) (gnu packages cross-base) (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages gnome) + (gnu packages image) + (gnu packages imagemagick) (gnu packages installers) (gnu packages linux) + (gnu packages llvm) (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) (gnu packages version-control) + (guix build-system font) (guix build-system gnu) (guix build-system trivial) + (guix download) (guix gexp) + ((guix licenses) #:prefix license:) (guix packages) (guix profiles) (guix utils)) +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + (define (make-ssp-fixed-gcc xgcc) "Given a XGCC package, return a modified package that uses the SSP function from glibc instead of from libssp.so. Our `symbol-check' script will complain if @@ -150,11 +166,33 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) +(define (make-nsis-with-sde-support base-nsis) + (package-with-extra-patches base-nsis + (search-our-patches "nsis-SConstruct-sde-support.patch"))) + +(define-public font-tuffy + (package + (name "font-tuffy") + (version "20120614") + (source + (origin + (method url-fetch) + (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) + (file-name (string-append name "-" version ".tar.gz")) + (sha256 + (base32 + "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) + (build-system font-build-system) + (home-page "http://tulrich.com/fonts/") + (synopsis "The Tuffy Truetype Font Family") + (description + "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") + (license license:public-domain))) (packages->manifest (append (list ;; The Basics - bash-minimal + bash which coreutils util-linux @@ -184,15 +222,19 @@ chain for " target " development.")) python-3.7 ;; Git git - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27)) + ;; Native gcc 7 toolchain + gcc-toolchain-7) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows - (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) + (list zip + (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (make-nsis-with-sde-support nsis-x86_64))) ((string-contains target "riscv64-linux-") (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" #:base-gcc-for-libc gcc-7))) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) + ((string-contains target "darwin") + (list clang-8 libcap binutils imagemagick libtiff librsvg font-tuffy cmake-3.15.5 xorriso)) (else '()))))) diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch new file mode 100644 index 0000000000..5edf1b7c8e --- /dev/null +++ b/contrib/guix/patches/nsis-SConstruct-sde-support.patch @@ -0,0 +1,15 @@ +diff --git a/SConstruct b/SConstruct +index e8252c9..41786f2 100755 +--- a/SConstruct ++++ b/SConstruct +@@ -95,8 +95,8 @@ default_doctype = 'html' + if defenv.WhereIs('hhc', os.environ['PATH']): + default_doctype = 'chm' + +-from time import strftime, gmtime +-cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) ++import time ++cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) + + opts = Variables() + diff --git a/contrib/init/README.md b/contrib/init/README.md index 306a37f75a..affc7c2e75 100644 --- a/contrib/init/README.md +++ b/contrib/init/README.md @@ -1,6 +1,6 @@ Sample configuration files for: ``` -SystemD: bitcoind.service +systemd: bitcoind.service Upstart: bitcoind.conf OpenRC: bitcoind.openrc bitcoind.openrcconf @@ -9,4 +9,4 @@ macOS: org.bitcoin.bitcoind.plist ``` have been made available to assist packagers in creating node packages here. -See doc/init.md for more information. +See [doc/init.md](../../doc/init.md) for more information. diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 8b308644b1..5999928aa4 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -11,7 +11,11 @@ [Unit] Description=Bitcoin daemon -After=network.target +Documentation=https://github.com/bitcoin/bitcoin/blob/master/doc/init.md + +# https://www.freedesktop.org/wiki/Software/systemd/NetworkTarget/ +After=network-online.target +Wants=network-online.target [Service] ExecStart=/usr/bin/bitcoind -daemon \ diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index e9130a21de..4037936404 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -68,10 +68,155 @@ tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" cd "${BDB_PREFIX}/${BDB_VERSION}/" # Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) -CLANG_CXX11_PATCH_URL='https://gist.githubusercontent.com/LnL7/5153b251fd525fe15de69b67e63a6075/raw/7778e9364679093a32dec2908656738e16b6bdcb/clang.patch' -CLANG_CXX11_PATCH_HASH='7a9a47b03fd5fb93a16ef42235fa9512db9b0829cfc3bdf90edd3ec1f44d637c' -http_get "${CLANG_CXX11_PATCH_URL}" clang.patch "${CLANG_CXX11_PATCH_HASH}" -patch -p2 < clang.patch +patch --ignore-whitespace -p1 << 'EOF' +commit 3311d68f11d1697565401eee6efc85c34f022ea7 +Author: fanquake <fanquake@gmail.com> +Date: Mon Aug 17 20:03:56 2020 +0800 + + Fix C++11 compatibility + +diff --git a/dbinc/atomic.h b/dbinc/atomic.h +index 0034dcc..7c11d4a 100644 +--- a/dbinc/atomic.h ++++ b/dbinc/atomic.h +@@ -70,7 +70,7 @@ typedef struct { + * These have no memory barriers; the caller must include them when necessary. + */ + #define atomic_read(p) ((p)->value) +-#define atomic_init(p, val) ((p)->value = (val)) ++#define atomic_init_db(p, val) ((p)->value = (val)) + + #ifdef HAVE_ATOMIC_SUPPORT + +@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; + #define atomic_inc(env, p) __atomic_inc(p) + #define atomic_dec(env, p) __atomic_dec(p) + #define atomic_compare_exchange(env, p, o, n) \ +- __atomic_compare_exchange((p), (o), (n)) ++ __atomic_compare_exchange_db((p), (o), (n)) + static inline int __atomic_inc(db_atomic_t *p) + { + int temp; +@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) + * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html + * which configure could be changed to use. + */ +-static inline int __atomic_compare_exchange( ++static inline int __atomic_compare_exchange_db( + db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) + { + atomic_value_t was; +@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( + #define atomic_dec(env, p) (--(p)->value) + #define atomic_compare_exchange(env, p, oldval, newval) \ + (DB_ASSERT(env, atomic_read(p) == (oldval)), \ +- atomic_init(p, (newval)), 1) ++ atomic_init_db(p, (newval)), 1) + #else + #define atomic_inc(env, p) __atomic_inc(env, p) + #define atomic_dec(env, p) __atomic_dec(env, p) +diff --git a/mp/mp_fget.c b/mp/mp_fget.c +index 5fdee5a..0b75f57 100644 +--- a/mp/mp_fget.c ++++ b/mp/mp_fget.c +@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ + + /* Initialize enough so we can call __memp_bhfree. */ + alloc_bhp->flags = 0; +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + #ifdef DIAGNOSTIC + if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { + __db_errx(env, +@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ + MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, + PROT_READ); + +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + MUTEX_LOCK(env, alloc_bhp->mtx_buf); + alloc_bhp->priority = bhp->priority; + alloc_bhp->pgno = bhp->pgno; +diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c +index 34467d2..f05aa0c 100644 +--- a/mp/mp_mvcc.c ++++ b/mp/mp_mvcc.c +@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) + #else + memcpy(frozen_bhp, bhp, SSZA(BH, buf)); + #endif +- atomic_init(&frozen_bhp->ref, 0); ++ atomic_init_db(&frozen_bhp->ref, 0); + if (mutex != MUTEX_INVALID) + frozen_bhp->mtx_buf = mutex; + else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, +@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) + #endif + alloc_bhp->mtx_buf = mutex; + MUTEX_LOCK(env, alloc_bhp->mtx_buf); +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + F_CLR(alloc_bhp, BH_FROZEN); + } + +diff --git a/mp/mp_region.c b/mp/mp_region.c +index e6cece9..ddbe906 100644 +--- a/mp/mp_region.c ++++ b/mp/mp_region.c +@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) + return (ret); + SH_TAILQ_INIT(&htab[i].hash_bucket); +- atomic_init(&htab[i].hash_page_dirty, 0); ++ atomic_init_db(&htab[i].hash_page_dirty, 0); + } + + /* +@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : + mtx_base + i; + SH_TAILQ_INIT(&hp->hash_bucket); +- atomic_init(&hp->hash_page_dirty, 0); ++ atomic_init_db(&hp->hash_page_dirty, 0); + #ifdef HAVE_STATISTICS + hp->hash_io_wait = 0; + hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; +diff --git a/mutex/mut_method.c b/mutex/mut_method.c +index 2588763..5c6d516 100644 +--- a/mutex/mut_method.c ++++ b/mutex/mut_method.c +@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) + MUTEX_LOCK(env, mtx); + ret = atomic_read(v) == oldval; + if (ret) +- atomic_init(v, newval); ++ atomic_init_db(v, newval); + MUTEX_UNLOCK(env, mtx); + + return (ret); +diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c +index f3922e0..e40fcdf 100644 +--- a/mutex/mut_tas.c ++++ b/mutex/mut_tas.c +@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) + + #ifdef HAVE_SHARED_LATCHES + if (F_ISSET(mutexp, DB_MUTEX_SHARED)) +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + else + #endif + if (MUTEX_INIT(&mutexp->tas)) { +@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) + F_CLR(mutexp, DB_MUTEX_LOCKED); + /* Flush flag update before zeroing count */ + MEMBAR_EXIT(); +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + } else { + DB_ASSERT(env, sharecount > 0); + MEMBAR_EXIT(); +EOF # The packaged config.guess and config.sub are ancient (2009) and can cause build issues. # Replace them with modern versions. diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index 5c5a85d3fe..d481413cc3 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -8,10 +8,9 @@ set -e UNSIGNED="$1" SIGNATURE="$2" -ARCH=x86_64 ROOTDIR=dist -TEMPDIR=signed.temp OUTDIR=signed-app +SIGNAPPLE=signapple if [ -z "$UNSIGNED" ]; then echo "usage: $0 <unsigned app> <signature>" @@ -23,35 +22,6 @@ if [ -z "$SIGNATURE" ]; then exit 1 fi -rm -rf ${TEMPDIR} && mkdir -p ${TEMPDIR} -tar -C ${TEMPDIR} -xf ${UNSIGNED} -cp -rf "${SIGNATURE}"/* ${TEMPDIR} - -if [ -z "${PAGESTUFF}" ]; then - PAGESTUFF=${TEMPDIR}/pagestuff -fi - -if [ -z "${CODESIGN_ALLOCATE}" ]; then - CODESIGN_ALLOCATE=${TEMPDIR}/codesign_allocate -fi - -find ${TEMPDIR} -name "*.sign" | while read i; do - SIZE=$(stat -c %s "${i}") - TARGET_FILE="$(echo "${i}" | sed 's/\.sign$//')" - - echo "Allocating space for the signature of size ${SIZE} in ${TARGET_FILE}" - ${CODESIGN_ALLOCATE} -i "${TARGET_FILE}" -a ${ARCH} ${SIZE} -o "${i}.tmp" - - OFFSET=$(${PAGESTUFF} "${i}.tmp" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') - if [ -z ${QUIET} ]; then - echo "Attaching signature at offset ${OFFSET}" - fi - - dd if="$i" of="${i}.tmp" bs=1 seek=${OFFSET} count=${SIZE} 2>/dev/null - mv "${i}.tmp" "${TARGET_FILE}" - rm "${i}" - echo "Success." -done -mv ${TEMPDIR}/${ROOTDIR} ${OUTDIR} -rm -rf ${TEMPDIR} +${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} +mv ${ROOTDIR} ${OUTDIR} echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 31a97f0a24..4f246cbb3f 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -8,44 +8,21 @@ set -e ROOTDIR=dist BUNDLE="${ROOTDIR}/Bitcoin-Qt.app" -CODESIGN=codesign +SIGNAPPLE=signapple TEMPDIR=sign.temp -TEMPLIST=${TEMPDIR}/signatures.txt OUT=signature-osx.tar.gz -OUTROOT=osx +OUTROOT=osx/dist if [ -z "$1" ]; then - echo "usage: $0 <codesign args>" - echo "example: $0 -s MyIdentity" + echo "usage: $0 <signapple args>" + echo "example: $0 <path to key>" exit 1 fi -rm -rf ${TEMPDIR} ${TEMPLIST} +rm -rf ${TEMPDIR} mkdir -p ${TEMPDIR} -${CODESIGN} -f --file-list ${TEMPLIST} "$@" "${BUNDLE}" - -grep -v CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" - SIZE=$(pagestuff "$i" -p | tail -2 | grep size | sed 's/[^0-9]*//g') - OFFSET=$(pagestuff "$i" -p | tail -2 | grep offset | sed 's/[^0-9]*//g') - SIGNFILE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}.sign" - DIRNAME="$(dirname "${SIGNFILE}")" - mkdir -p "${DIRNAME}" - echo "Adding detached signature for: ${TARGETFILE}. Size: ${SIZE}. Offset: ${OFFSET}" - dd if="$i" of="${SIGNFILE}" bs=1 skip=${OFFSET} count=${SIZE} 2>/dev/null -done - -grep CodeResources < "${TEMPLIST}" | while read i; do - TARGETFILE="${BUNDLE}/$(echo "${i}" | sed "s|.*${BUNDLE}/||")" - RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" - DIRNAME="$(dirname "${RESOURCE}")" - mkdir -p "${DIRNAME}" - echo "Adding resource for: \"${TARGETFILE}\"" - cp "${i}" "${RESOURCE}" -done - -rm ${TEMPLIST} +${SIGNAPPLE} sign -f --detach "${TEMPDIR}/${OUTROOT}" "$@" "${BUNDLE}" tar -C "${TEMPDIR}" -czf "${OUT}" . rm -rf "${TEMPDIR}" diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md new file mode 100644 index 0000000000..7301968461 --- /dev/null +++ b/contrib/message-capture/message-capture-docs.md @@ -0,0 +1,25 @@ +# Per-Peer Message Capture + +## Purpose + +This feature allows for message capture on a per-peer basis. It answers the simple question: "Can I see what messages my node is sending and receiving?" + +## Usage and Functionality + +* Run `bitcoind` with the `-capturemessages` option. +* Look in the `message_capture` folder in your datadir. + * Typically this will be `~/.bitcoin/message_capture`. + * See that there are many folders inside, one for each peer names with its IP address and port. + * Inside each peer's folder there are two `.dat` files: one is for received messages (`msgs_recv.dat`) and the other is for sent messages (`msgs_sent.dat`). +* Run `contrib/message-capture/message-capture-parser.py` with the proper arguments. + * See the `-h` option for help. + * To see all messages, both sent and received, for all peers use: + ``` + ./contrib/message-capture/message-capture-parser.py -o out.json \ + ~/.bitcoin/message_capture/**/*.dat + ``` + * Note: The messages in the given `.dat` files will be interleaved in chronological order. So, giving both received and sent `.dat` files (as above with `*.dat`) will result in all messages being interleaved in chronological order. + * If an output file is not provided (i.e. the `-o` option is not used), then the output prints to `stdout`. +* View the resulting output. + * The output file is `JSON` formatted. + * Suggestion: use `jq` to view the output, with `jq . out.json` diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py new file mode 100755 index 0000000000..9988478f1b --- /dev/null +++ b/contrib/message-capture/message-capture-parser.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Parse message capture binary files. To be used in conjunction with -capturemessages.""" + +import argparse +import os +import shutil +import sys +from io import BytesIO +import json +from pathlib import Path +from typing import Any, List, Optional + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.messages import ser_uint256 # noqa: E402 +from test_framework.p2p import MESSAGEMAP # noqa: E402 + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +# The test framework classes stores hashes as large ints in many cases. +# These are variables of type uint256 in core. +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. +# As such, they are itemized here. +# Any variables with these names that are of type int are actually uint256 variables. +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) +HASH_INTS = [ + "blockhash", + "block_hash", + "hash", + "hashMerkleRoot", + "hashPrevBlock", + "hashstop", + "prev_header", + "sha256", + "stop_hash", +] + +HASH_INT_VECTORS = [ + "hashes", + "headers", + "vHave", + "vHash", +] + + +class ProgressBar: + def __init__(self, total: float): + self.total = total + self.running = 0 + + def set_progress(self, progress: float): + cols = shutil.get_terminal_size()[0] + if cols <= 12: + return + max_blocks = cols - 9 + num_blocks = int(max_blocks * progress) + print('\r[ {}{} ] {:3.0f}%' + .format('#' * num_blocks, + ' ' * (max_blocks - num_blocks), + progress * 100), + end ='') + + def update(self, more: float): + self.running += more + self.set_progress(self.running / self.total) + + +def to_jsonable(obj: Any) -> Any: + if hasattr(obj, "__dict__"): + return obj.__dict__ + elif hasattr(obj, "__slots__"): + ret = {} # type: Any + for slot in obj.__slots__: + val = getattr(obj, slot, None) + if slot in HASH_INTS and isinstance(val, int): + ret[slot] = ser_uint256(val).hex() + elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + ret[slot] = [ser_uint256(a).hex() for a in val] + else: + ret[slot] = to_jsonable(val) + return ret + elif isinstance(obj, list): + return [to_jsonable(a) for a in obj] + elif isinstance(obj, bytes): + return obj.hex() + else: + return obj + + +def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: + with open(path, 'rb') as f_in: + if progress_bar: + bytes_read = 0 + + while True: + if progress_bar: + # Update progress bar + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + bytes_read = f_in.tell() - 1 + + # Read the Header + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + msgtype = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] # type: bytes + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + + # Start converting the message to a dictionary + msg_dict = {} + msg_dict["direction"] = "recv" if recv else "sent" + msg_dict["time"] = time + msg_dict["size"] = length # "size" is less readable here, but more readable in the output + + msg_ser = BytesIO(f_in.read(length)) + + # Determine message type + if msgtype not in MESSAGEMAP: + # Unrecognized message type + try: + msgtype_tmp = msgtype.decode() + if not msgtype_tmp.isprintable(): + raise UnicodeDecodeError + msg_dict["msgtype"] = msgtype_tmp + except UnicodeDecodeError: + msg_dict["msgtype"] = "UNREADABLE" + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unrecognized message type." + messages.append(msg_dict) + print(f"WARNING - Unrecognized message type {msgtype} in {path}", file=sys.stderr) + continue + + # Deserialize the message + msg = MESSAGEMAP[msgtype]() + msg_dict["msgtype"] = msgtype.decode() + + try: + msg.deserialize(msg_ser) + except KeyboardInterrupt: + raise + except Exception: + # Unable to deserialize message body + msg_ser.seek(0, os.SEEK_SET) + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unable to deserialize message." + messages.append(msg_dict) + print(f"WARNING - Unable to deserialize message in {path}", file=sys.stderr) + continue + + # Convert body of message into a jsonable object + if length: + msg_dict["body"] = to_jsonable(msg) + messages.append(msg_dict) + + if progress_bar: + # Update the progress bar to the end of the current file + # in case we exited the loop early + f_in.seek(0, os.SEEK_END) # Go to end of file + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format(sys.argv[0]), + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "capturepaths", + nargs='+', + help="binary message capture files to parse.") + parser.add_argument( + "-o", "--output", + help="output file. If unset print to stdout") + parser.add_argument( + "-n", "--no-progress-bar", + action='store_true', + help="disable the progress bar. Automatically set if the output is not a terminal") + args = parser.parse_args() + capturepaths = [Path.cwd() / Path(capturepath) for capturepath in args.capturepaths] + output = Path.cwd() / Path(args.output) if args.output else False + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() + + messages = [] # type: List[Any] + if use_progress_bar: + total_size = sum(capture.stat().st_size for capture in capturepaths) + progress_bar = ProgressBar(total_size) + else: + progress_bar = None + + for capture in capturepaths: + process_file(str(capture), messages, "recv" in capture.stem, progress_bar) + + messages.sort(key=lambda msg: msg['time']) + + if use_progress_bar: + progress_bar.set_progress(1) + + jsonrep = json.dumps(messages) + if output: + with open(str(output), 'w+', encoding="utf8") as f_out: + f_out.write(jsonrep) + else: + print(jsonrep) + +if __name__ == "__main__": + main() diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 4209fdb364..c50d4bef71 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -21,21 +21,21 @@ The script returns 0 if everything passes the checks. It returns 1 if either the ```sh -./verify.sh bitcoin-core-0.11.2 -./verify.sh bitcoin-core-0.12.0 -./verify.sh bitcoin-core-0.13.0-rc3 +./verify.py bitcoin-core-0.11.2 +./verify.py bitcoin-core-0.12.0 +./verify.py bitcoin-core-0.13.0-rc3 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./verify.sh bitcoin-core-0.11.2-osx -./verify.sh 0.12.0-linux -./verify.sh bitcoin-core-0.13.0-rc3-win64 +./verify.py bitcoin-core-0.11.2-osx +./verify.py 0.12.0-linux +./verify.py bitcoin-core-0.13.0-rc3-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./verify.sh bitcoin-core-0.13.0 delete +./verify.py bitcoin-core-0.13.0 delete ``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py new file mode 100755 index 0000000000..b97ad11b8e --- /dev/null +++ b/contrib/verifybinaries/verify.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitoin Core release binaries + +This script attempts to download the signature file SHA256SUMS.asc from +bitcoincore.org and bitcoin.org and compares them. +It first checks if the signature passes, and then downloads the files +specified in the file, and checks if the hashes of these files match those +that are specified in the signature file. +The script returns 0 if everything passes the checks. It returns 1 if either +the signature check or the hash check doesn't pass. If an error occurs the +return value is >= 2. +""" +from hashlib import sha256 +import os +import subprocess +import sys +from textwrap import indent + +WORKINGDIR = "/tmp/bitcoin_verify_binaries" +HASHFILE = "hashes.tmp" +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SIGNATUREFILENAME = "SHA256SUMS.asc" + + +def parse_version_string(version_str): + if version_str.startswith(VERSIONPREFIX): # remove version prefix + version_str = version_str[len(VERSIONPREFIX):] + + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "<version>-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "<version>-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file=None): + if local_file: + wget_args = ['wget', '-O', local_file, remote_file] + else: + # use timestamping mechanism if local filename is not explicitely set + wget_args = ['wget', '-N', remote_file] + + result = subprocess.run(wget_args, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + return contents1 == contents2 + + +def verify_with_gpg(signature_filename, output_filename): + result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', + output_filename, signature_filename], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode, result.stdout.decode().rstrip() + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +def main(args): + # sanity check + if len(args) < 1: + print("Error: need to specify a version on the command line") + return 3 + + # determine remote dir dependend on provided version string + version_base, version_rc, os_filter = parse_version_string(args[0]) + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigfile = remote_dir + SIGNATUREFILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + # fetch first signature file + sigfile1 = SIGNATUREFILENAME + success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) + if not success: + print("Error: couldn't fetch signature file. " + "Have you specified the version number in the following format?") + print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] " + f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") + print("wget output:") + print(indent(output, '\t')) + return 4 + + # fetch second signature file + sigfile2 = SIGNATUREFILENAME + ".2" + success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) + if not success: + print("bitcoin.org failed to provide signature file, " + "but bitcoincore.org did?") + print("wget output:") + print(indent(output, '\t')) + remove_files([sigfile1]) + return 5 + + # ensure that both signature files are equal + if not files_are_equal(sigfile1, sigfile2): + print("bitcoin.org and bitcoincore.org signature files were not equal?") + print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") + return 6 + + # check signature and extract data into file + retval, output = verify_with_gpg(sigfile1, HASHFILE) + if retval != 0: + if retval == 1: + print("Bad signature.") + elif retval == 2: + print("gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + print("gpg output:") + print(indent(output, '\t')) + remove_files([sigfile1, sigfile2, HASHFILE]) + return 1 + + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: "<hash> <binary_filename>" + with open(HASHFILE, 'r', encoding='utf8') as hash_file: + hashes_to_verify = [ + line.split()[:2] for line in hash_file if os_filter in line] + remove_files([HASHFILE]) + if not hashes_to_verify: + print("error: no files matched the platform specified") + return 7 + + # download binaries + for _, binary_filename in hashes_to_verify: + print(f"Downloading {binary_filename}") + download_with_wget(HOST1 + remote_dir + binary_filename) + + # verify hashes + offending_files = [] + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + if offending_files: + print("Hashes don't match.") + print("Offending files:") + print('\n'.join(offending_files)) + return 1 + verified_binaries = [entry[1] for entry in hashes_to_verify] + + # clean up files if desired + if len(args) >= 2: + print("Clean up the binaries") + remove_files([sigfile1, sigfile2] + verified_binaries) + else: + print(f"Keep the binaries in {WORKINGDIR}") + + print("Verified hashes of") + print('\n'.join(verified_binaries)) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh deleted file mode 100755 index 4296998631..0000000000 --- a/contrib/verifybinaries/verify.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -### This script attempts to download the signature file SHA256SUMS.asc from -### bitcoincore.org and bitcoin.org and compares them. -### It first checks if the signature passes, and then downloads the files specified in -### the file, and checks if the hashes of these files match those that are specified -### in the signature file. -### The script returns 0 if everything passes the checks. It returns 1 if either the -### signature check or the hash check doesn't pass. If an error occurs the return value is 2 - -export LC_ALL=C -function clean_up { - for file in "$@" - do - rm "$file" 2> /dev/null - done -} - -WORKINGDIR="/tmp/bitcoin_verify_binaries" -TMPFILE="hashes.tmp" - -SIGNATUREFILENAME="SHA256SUMS.asc" -RCSUBDIR="test" -HOST1="https://bitcoincore.org" -HOST2="https://bitcoin.org" -BASEDIR="/bin/" -VERSIONPREFIX="bitcoin-core-" -RCVERSIONSTRING="rc" - -if [ ! -d "$WORKINGDIR" ]; then - mkdir "$WORKINGDIR" -fi - -cd "$WORKINGDIR" || exit 1 - -#test if a version number has been passed as an argument -if [ -n "$1" ]; then - #let's also check if the version number includes the prefix 'bitcoin-', - # and add this prefix if it doesn't - if [[ $1 == "$VERSIONPREFIX"* ]]; then - VERSION="$1" - else - VERSION="$VERSIONPREFIX$1" - fi - - STRIPPEDLAST="${VERSION%-*}" - - #now let's see if the version string contains "rc" or a platform name (e.g. "osx") - if [[ "$STRIPPEDLAST-" == "$VERSIONPREFIX" ]]; then - BASEDIR="$BASEDIR$VERSION/" - else - # let's examine the last part to see if it's rc and/or platform name - STRIPPEDNEXTTOLAST="${STRIPPEDLAST%-*}" - if [[ "$STRIPPEDNEXTTOLAST-" == "$VERSIONPREFIX" ]]; then - - LASTSUFFIX="${VERSION##*-}" - VERSION="$STRIPPEDLAST" - - if [[ $LASTSUFFIX == *"$RCVERSIONSTRING"* ]]; then - RCVERSION="$LASTSUFFIX" - else - PLATFORM="$LASTSUFFIX" - fi - - else - RCVERSION="${STRIPPEDLAST##*-}" - PLATFORM="${VERSION##*-}" - - VERSION="$STRIPPEDNEXTTOLAST" - fi - - BASEDIR="$BASEDIR$VERSION/" - if [[ $RCVERSION == *"$RCVERSIONSTRING"* ]]; then - BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/" - fi - fi -else - echo "Error: need to specify a version on the command line" - exit 2 -fi - -if ! WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" - # shellcheck disable=SC1087 - echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - exit 2 -fi - -if ! WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME - exit 3 -fi - -SIGFILEDIFFS="$(diff $SIGNATUREFILENAME $SIGNATUREFILENAME.2)" -if [ "$SIGFILEDIFFS" != "" ]; then - echo "bitcoin.org and bitcoincore.org signature files were not equal?" - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 - exit 4 -fi - -#then we check it -GPGOUT=$(gpg --yes --decrypt --output "$TMPFILE" "$SIGNATUREFILENAME" 2>&1) - -#return value 0: good signature -#return value 1: bad signature -#return value 2: gpg error - -RET="$?" -if [ $RET -ne 0 ]; then - if [ $RET -eq 1 ]; then - #and notify the user if it's bad - echo "Bad signature." - elif [ $RET -eq 2 ]; then - #or if a gpg error has occurred - echo "gpg error. Do you have the Bitcoin Core binary release signing key installed?" - fi - - echo "gpg output:" - # shellcheck disable=SC2001 - echo "$GPGOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE - exit "$RET" -fi - -if [ -n "$PLATFORM" ]; then - grep $PLATFORM $TMPFILE > "$TMPFILE-plat" - TMPFILESIZE=$(stat -c%s "$TMPFILE-plat") - if [ $TMPFILESIZE -eq 0 ]; then - echo "error: no files matched the platform specified" && exit 3 - fi - mv "$TMPFILE-plat" $TMPFILE -fi - -#here we extract the filenames from the signature file -FILES=$(awk '{print $2}' "$TMPFILE") - -#and download these one by one -for file in $FILES -do - echo "Downloading $file" - wget --quiet -N "$HOST1$BASEDIR$file" -done - -#check hashes -DIFF=$(diff <(sha256sum $FILES) "$TMPFILE") - -if [ $? -eq 1 ]; then - echo "Hashes don't match." - echo "Offending files:" - echo "$DIFF"|grep "^<"|awk '{print "\t"$3}' - exit 1 -elif [ $? -gt 1 ]; then - echo "Error executing 'diff'" - exit 2 -fi - -if [ -n "$2" ]; then - echo "Clean up the binaries" - clean_up $FILES $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE -else - echo "Keep the binaries in $WORKINGDIR" - clean_up $TMPFILE -fi - -echo -e "Verified hashes of \n$FILES" - -exit 0 diff --git a/depends/Makefile b/depends/Makefile index 596a46d4a2..016b73a5cb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -265,7 +265,7 @@ install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin14 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: diff --git a/depends/README.md b/depends/README.md index 9bc9b6714b..9e9878c595 100644 --- a/depends/README.md +++ b/depends/README.md @@ -44,7 +44,7 @@ The paths are automatically configured and no other options are needed unless ta #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5 + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5 xorriso #### For Win64 cross compilation diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 6237734390..ea06952af4 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -111,12 +111,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 Returns various information about the TX mempool. Only supports JSON as output format. -* loaded : (boolean) if the mempool is fully loaded -* size : (numeric) the number of transactions in the TX mempool -* bytes : (numeric) size of the TX mempool in bytes -* usage : (numeric) total TX mempool memory usage -* maxmempool : (numeric) maximum memory usage for the mempool in bytes -* mempoolminfee : (numeric) minimum feerate (BTC per KB) for tx to be accepted +Refer to the `getmempoolinfo` RPC for documentation of the fields. `GET /rest/mempool/contents.json` diff --git a/doc/build-unix.md b/doc/build-unix.md index 5c24886dbf..4051440886 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -9,7 +9,7 @@ Note Always use absolute paths to configure and compile Bitcoin Core and the dependencies. For example, when specifying the path of the dependency: - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX + ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures the usage of the absolute path. @@ -166,9 +166,9 @@ miniupnpc https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and turned off by default. See the configure options for UPnP behavior desired: - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime + --without-miniupnpc No UPnP support, miniupnp not required + --disable-upnp-default (the default) UPnP support turned off by default at runtime + --enable-upnp-default UPnP support turned on by default at runtime libnatpmp --------- @@ -177,9 +177,9 @@ libnatpmp from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and turned off by default. See the configure options for NAT-PMP behavior desired: - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime + --without-natpmp No NAT-PMP support, libnatpmp not required + --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime + --enable-natpmp-default NAT-PMP support turned on by default at runtime Berkeley DB ----------- @@ -199,9 +199,9 @@ Boost ----- If you need to build Boost yourself: - sudo su - ./bootstrap.sh - ./bjam install + sudo su + ./bootstrap.sh + ./bjam install Security @@ -212,8 +212,8 @@ This can be disabled with: Hardening Flags: - ./configure --enable-hardening - ./configure --disable-hardening + ./configure --enable-hardening + ./configure --disable-hardening Hardening enables the following features: @@ -228,7 +228,7 @@ Hardening enables the following features: To test that you have built PIE executable, install scanelf, part of paxutils, and use: - scanelf -e ./bitcoin + scanelf -e ./bitcoin The output should contain: @@ -245,8 +245,8 @@ Hardening enables the following features: `scanelf -e ./bitcoin` The output should contain: - STK/REL/PTL - RW- R-- RW- + STK/REL/PTL + RW- R-- RW- The STK RW- means that the stack is readable and writeable but not executable. diff --git a/doc/descriptors.md b/doc/descriptors.md index 63acb9167f..c4fc2a66bf 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -191,7 +191,7 @@ steps, or for dumping wallet descriptors including private key material. In order to easily represent the sets of scripts currently supported by existing Bitcoin Core wallets, a convenience function `combo` is provided, which takes as input a public key, and describes a set of P2PK, -P2PKH, P2WPKH, and P2SH-P2WPH scripts for that key. In case the key is +P2PKH, P2WPKH, and P2SH-P2WPKH scripts for that key. In case the key is uncompressed, the set only includes P2PK and P2PKH scripts. ### Checksums diff --git a/doc/init.md b/doc/init.md index 99aa0a0def..399b819bf4 100644 --- a/doc/init.md +++ b/doc/init.md @@ -53,11 +53,12 @@ Paths All three configurations assume several paths that might need to be adjusted. -Binary: `/usr/bin/bitcoind` -Configuration file: `/etc/bitcoin/bitcoin.conf` -Data directory: `/var/lib/bitcoind` -PID file: `/var/run/bitcoind/bitcoind.pid` (OpenRC and Upstart) or `/run/bitcoind/bitcoind.pid` (systemd) -Lock file: `/var/lock/subsys/bitcoind` (CentOS) + Binary: /usr/bin/bitcoind + Configuration file: /etc/bitcoin/bitcoin.conf + Data directory: /var/lib/bitcoind + PID file: /var/run/bitcoind/bitcoind.pid (OpenRC and Upstart) or + /run/bitcoind/bitcoind.pid (systemd) + Lock file: /var/lock/subsys/bitcoind (CentOS) The PID directory (if applicable) and data directory should both be owned by the bitcoin user and group. It is advised for security reasons to make the @@ -83,10 +84,10 @@ OpenRC). ### macOS -Binary: `/usr/local/bin/bitcoind` -Configuration file: `~/Library/Application Support/Bitcoin/bitcoin.conf` -Data directory: `~/Library/Application Support/Bitcoin` -Lock file: `~/Library/Application Support/Bitcoin/.lock` + Binary: /usr/local/bin/bitcoind + Configuration file: ~/Library/Application Support/Bitcoin/bitcoin.conf + Data directory: ~/Library/Application Support/Bitcoin + Lock file: ~/Library/Application Support/Bitcoin/.lock Installing Service Configuration ----------------------------------- diff --git a/doc/release-notes.md b/doc/release-notes.md index 8f1e03e16b..0f248494c7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -93,6 +93,10 @@ Tools and Utilities Wallet ------ +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + GUI changes ----------- diff --git a/doc/release-notes/release-notes-0.21.0.md b/doc/release-notes/release-notes-0.21.0.md index 66aee77643..3baba3d49b 100644 --- a/doc/release-notes/release-notes-0.21.0.md +++ b/doc/release-notes/release-notes-0.21.0.md @@ -100,7 +100,7 @@ P2P and network changes - This release adds support for serving [BIP157](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) compact filters to peers on the network when enabled using - `-blockfilterindex=1 -peercfilters=1`. (#16442) + `-blockfilterindex=1 -peerblockfilters=1`. (#16442) - This release adds support for signets ([BIP325](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki)) in diff --git a/doc/release-process.md b/doc/release-process.md index 92845bcc82..84b208a0d8 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -5,7 +5,7 @@ Release Process ### Before every release candidate -* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). +* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). @@ -52,6 +52,13 @@ Release Process - Merge the release notes from the wiki into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. +#### Tagging a release (candidate) + +To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: + + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + +This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. ## Building @@ -73,21 +80,12 @@ Open a draft of the release notes for collaborative editing at https://github.co For the period during which the notes are being edited on the wiki, the version on the branch should be wiped and replaced with a link to the wiki which should be used for all announcements until `-final`. -Write the release notes. `git shortlog` helps a lot, for example: - - git shortlog --no-merges v(current version, e.g. 0.19.2)..v(new version, e.g. 0.20.0) - -(or ping @wumpus on IRC, he has specific tooling to generate the list of merged pulls -and sort them into categories based on labels). +Generate the change log. As this is a huge amount of work to do manually, there is the `list-pulls` script to do a pre-sorting step based on github PR metadata. See the [documentation in the README.md](https://github.com/bitcoin-core/bitcoin-maintainer-tools/blob/master/README.md#list-pulls). Generate list of authors: git log --format='- %aN' v(current version, e.g. 0.20.0)..v(new version, e.g. 0.20.1) | sort -fiu -Tag the version (or release candidate) in git: - - git tag -s v(new version, e.g. 0.20.0) - ### Setup and perform Gitian builds If you're using the automated script (found in [contrib/gitian-build.py](/contrib/gitian-build.py)), then at this point you should run it with the "--build" command. Otherwise ignore this. @@ -326,6 +324,18 @@ bitcoin.org (see below for bitcoin.org update instructions). - bitcoincore.org RPC documentation update + - Install [golang](https://golang.org/doc/install) + + - Install the new Bitcoin Core release + + - Run bitcoind on regtest + + - Clone the [bitcoincore.org repository](https://github.com/bitcoin-core/bitcoincore.org) + + - Run: `go run generate.go` while being in `contrib/doc-gen` folder, and with bitcoin-cli in PATH + + - Add the generated files to git + - Update packaging repo - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 diff --git a/doc/tor.md b/doc/tor.md index 1ba7137b8e..e38ada5bd6 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -21,129 +21,193 @@ information in the debug log about your Tor configuration. The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all outgoing connections, but more is possible. - -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy - server will be used to try to reach .onion addresses as well. + -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy + server will be used to try to reach .onion addresses as well. + You need to use -noonion or -onion=0 to explicitly disable + outbound access to onion services. + + -onion=ip:port Set the proxy server to use for Tor onion services. You do not + need to set this if it's the same as -proxy. You can use -onion=0 + to explicitly disable access to onion services. + Note: Only the -proxy option sets the proxy for DNS requests; + with -onion they will not route over Tor, so use -proxy if you + have privacy concerns. + + -listen When using -proxy, listening is disabled by default. If you want + to manually configure an onion service (see section 3), you'll + need to enable it explicitly. + + -connect=X When behind a Tor proxy, you can specify .onion addresses instead + -addnode=X of IP addresses or hostnames in these parameters. It requires + -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with + other P2P nodes. + + -onlynet=onion Make outgoing connections only to .onion addresses. Incoming + connections are not affected by this option. This option can be + specified multiple times to allow multiple network types, e.g. + ipv4, ipv6 or onion. If you use this option with values other + than onion you *cannot* disable onion connections; outgoing onion + connections will be enabled when you use -proxy or -onion. Use + -noonion or -onion=0 if you want to be sure there are no outbound + onion connections over the default proxy or your defined -proxy. - -onion=ip:port Set the proxy server to use for Tor onion services. You do not - need to set this if it's the same as -proxy. You can use -noonion - to explicitly disable access to onion services. +In a typical situation, this suffices to run behind a Tor proxy: - -listen When using -proxy, listening is disabled by default. If you want - to run an onion service (see next section), you'll need to enable - it explicitly. + ./bitcoind -proxy=127.0.0.1:9050 - -connect=X When behind a Tor proxy, you can specify .onion addresses instead - -addnode=X of IP addresses or hostnames in these parameters. It requires - -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with - other P2P nodes. +## 2. Automatically create a Bitcoin Core onion service - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - ipv4, ipv6, or onion. +Bitcoin Core makes use of Tor's control socket API to create and destroy +ephemeral onion services programmatically. This means that if Tor is running and +proper authentication has been configured, Bitcoin Core automatically creates an +onion service to listen on. The goal is to increase the number of available +onion nodes. -In a typical situation, this suffices to run behind a Tor proxy: +This feature is enabled by default if Bitcoin Core is listening (`-listen`) and +it requires a Tor connection to work. It can be explicitly disabled with +`-listenonion=0`. If it is not disabled, it can be configured using the +`-torcontrol` and `-torpassword` settings. + +To see verbose Tor information in the bitcoind debug log, pass `-debug=tor`. + +### Control Port + +You may need to set up the Tor Control Port. On Linux distributions there may be +some or all of the following settings in `/etc/tor/torrc`, generally commented +out by default (if not, add them): + +``` +ControlPort 9051 +CookieAuthentication 1 +CookieAuthFileGroupReadable 1 +``` + +Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` +or `sudo systemctl restart tor` on most systemd-based systems, including recent +Debian and Ubuntu, or just restart the computer). + +On some systems (such as Arch Linux), you may also need to add the following +line: + +``` +DataDirectoryGroupReadable 1 +``` + +### Authentication + +Connecting to Tor's control socket API requires one of two authentication +methods to be configured: cookie authentication or bitcoind's `-torpassword` +configuration option. + +#### Cookie authentication + +For cookie authentication, the user running bitcoind must have read access to +the `CookieAuthFile` specified in the Tor configuration. In some cases this is +preconfigured and the creation of an onion service is automatic. Don't forget to +use the `-debug=tor` bitcoind configuration option to enable Tor debug logging. + +If a permissions problem is seen in the debug log, e.g. `tor: Authentication +cookie /run/tor/control.authcookie could not be opened (check permissions)`, it +can be resolved by adding both the user running Tor and the user running +bitcoind to the same Tor group and setting permissions appropriately. + +On Debian-derived systems, the Tor group will likely be `debian-tor` and one way +to verify could be to list the groups and grep for a "tor" group name: + +``` +getent group | cut -d: -f1 | grep -i tor +``` + +You can also check the group of the cookie file. On most Linux systems, the Tor +auth cookie will usually be `/run/tor/control.authcookie`: + +``` +stat -c '%G' /run/tor/control.authcookie +``` + +Once you have determined the `${TORGROUP}` and selected the `${USER}` that will +run bitcoind, run this as root: + +``` +usermod -a -G ${TORGROUP} ${USER} +``` + +Then restart the computer (or log out) and log in as the `${USER}` that will run +bitcoind. + +#### `torpassword` authentication + +For the `-torpassword=password` option, the password is the clear text form that +was used when generating the hashed password for the `HashedControlPassword` +option in the Tor configuration file. - ./bitcoind -proxy=127.0.0.1:9050 +The hashed password can be obtained with the command `tor --hash-password +password` (refer to the [Tor Dev +Manual](https://2019.www.torproject.org/docs/tor-manual.html.en) for more +details). -## 2. Manually create a Bitcoin Core onion service +## 3. Manually create a Bitcoin Core onion service -If you configure your Tor system accordingly, it is possible to make your node also -reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent -config file): *Needed for Tor version 0.2.7.0 and older versions of Tor only. For newer -versions of Tor see [Section 3](#3-automatically-listen-on-tor).* +You can also manually configure your node to be reachable from the Tor network. +Add these lines to your `/etc/tor/torrc` (or equivalent config file): - HiddenServiceDir /var/lib/tor/bitcoin-service/ - HiddenServicePort 8333 127.0.0.1:8334 + HiddenServiceDir /var/lib/tor/bitcoin-service/ + HiddenServicePort 8333 127.0.0.1:8334 The directory can be different of course, but virtual port numbers should be equal to your bitcoind's P2P listen port (8333 by default), and target addresses and ports should be equal to binding address and port for inbound Tor connections (127.0.0.1:8334 by default). - -externalip=X You can tell bitcoin about its publicly reachable addresses using - this option, and this can be an onion address. Given the above - configuration, you can find your onion address in - /var/lib/tor/bitcoin-service/hostname. For connections - coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), onion addresses are given - preference for your node to advertise itself with. - - You can set multiple local addresses with -externalip. The - one that will be rumoured to a particular peer is the most - compatible one and also using heuristics, e.g. the address - with the most incoming connections, etc. - - -listen You'll need to enable listening for incoming connections, as this - is off by default behind a proxy. - - -discover When -externalip is specified, no attempt is made to discover local - IPv4 or IPv6 addresses. If you want to run a dual stack, reachable - from both Tor and IPv4 (or IPv6), you'll need to either pass your - other addresses using -externalip, or explicitly enable -discover. - Note that both addresses of a dual-stack system may be easily - linkable using traffic analysis. + -externalip=X You can tell bitcoin about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in + /var/lib/tor/bitcoin-service/hostname. For connections + coming from unroutable addresses (such as 127.0.0.1, where the + Tor proxy typically runs), onion addresses are given + preference for your node to advertise itself with. + + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + + -listen You'll need to enable listening for incoming connections, as this + is off by default behind a proxy. + + -discover When -externalip is specified, no attempt is made to discover local + IPv4 or IPv6 addresses. If you want to run a dual stack, reachable + from both Tor and IPv4 (or IPv6), you'll need to either pass your + other addresses using -externalip, or explicitly enable -discover. + Note that both addresses of a dual-stack system may be easily + linkable using traffic analysis. In a typical situation, where you're only reachable via Tor, this should suffice: - ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen + ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing your address. To mitigate this, additionally bind the address of your Tor proxy: - ./bitcoind ... -bind=127.0.0.1 + ./bitcoind ... -bind=127.0.0.1 If you don't care too much about hiding your node, and want to be reachable on IPv4 as well, use `discover` instead: - ./bitcoind ... -discover + ./bitcoind ... -discover and open port 8333 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover - -## 3. Automatically create a Bitcoin Core onion service - -Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket -API, to create and destroy 'ephemeral' onion services programmatically. -Bitcoin Core has been updated to make use of this. - -This means that if Tor is running (and proper authentication has been configured), -Bitcoin Core automatically creates an onion service to listen on. This will positively -affect the number of available .onion nodes. - -This new feature is enabled by default if Bitcoin Core is listening (`-listen`), and -requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0` -and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings. -To show verbose debugging information, pass `-debug=tor`. - -Connecting to Tor's control socket API requires one of two authentication methods to be -configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051` -in `torrc` config file. For cookie authentication the user running bitcoind must have read -access to the `CookieAuthFile` specified in Tor configuration. In some cases this is -preconfigured and the creation of an onion service is automatic. If permission problems -are seen with `-debug=tor` they can be resolved by adding both the user running Tor and -the user running bitcoind to the same group and setting permissions appropriately. On -Debian-based systems the user running bitcoind can be added to the debian-tor group, -which has the appropriate permissions. Before starting bitcoind you will need to re-login -to allow debian-tor group to be applied. Otherwise you will see the following notice: "tor: -Authentication cookie /run/tor/control.authcookie could not be opened (check permissions)" -on debug.log. - -An alternative authentication method is the use -of the `-torpassword=password` option. The `password` is the clear text form that -was used when generating the hashed password for the `HashedControlPassword` option -in the tor configuration file. The hashed password can be obtained with the command -`tor --hash-password password` (read the tor manual for more details). + ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover ## 4. Privacy recommendations -- Do not add anything but Bitcoin Core ports to the onion service created in section 2. +- Do not add anything but Bitcoin Core ports to the onion service created in section 3. If you run a web service too, create a new onion service for that. Otherwise it is trivial to link them, which may reduce privacy. Onion - services created automatically (as in section 3) always have only one port + services created automatically (as in section 2) always have only one port open. diff --git a/src/Makefile.am b/src/Makefile.am index 2e35ecdfbd..bc661fccbb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,6 +2,10 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +# Pattern rule to print variables, e.g. make print-top_srcdir +print-%: + @echo $* = $($*) + DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) @@ -229,6 +233,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/error.h \ util/fees.h \ + util/getuniquepath.h \ util/golombrice.h \ util/hasher.h \ util/macros.h \ @@ -244,6 +249,7 @@ BITCOIN_CORE_H = \ util/system.h \ util/threadnames.h \ util/time.h \ + util/trace.h \ util/translation.h \ util/ui_change_type.h \ util/url.h \ @@ -552,6 +558,7 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/error.cpp \ util/fees.cpp \ + util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ util/system.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 3d41d203d3..969f0ca411 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -78,6 +78,7 @@ QT_MOC_CPP = \ qt/moc_transactiondesc.cpp \ qt/moc_transactiondescdialog.cpp \ qt/moc_transactionfilterproxy.cpp \ + qt/moc_transactionoverviewwidget.cpp \ qt/moc_transactiontablemodel.cpp \ qt/moc_transactionview.cpp \ qt/moc_utilitydialog.cpp \ @@ -151,6 +152,7 @@ BITCOIN_QT_H = \ qt/transactiondesc.h \ qt/transactiondescdialog.h \ qt/transactionfilterproxy.h \ + qt/transactionoverviewwidget.h \ qt/transactionrecord.h \ qt/transactiontablemodel.h \ qt/transactionview.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 770aba467d..e817bb2ee2 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,9 +2,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -if ENABLE_FUZZ +if ENABLE_FUZZ_BINARY noinst_PROGRAMS += test/fuzz/fuzz -else +endif + +if !ENABLE_FUZZ bin_PROGRAMS += test/test_bitcoin endif @@ -50,6 +52,14 @@ FUZZ_SUITE_LD_COMMON = \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) +if USE_UPNP +FUZZ_SUITE_LD_COMMON += $(MINIUPNPC_LIBS) +endif + +if USE_NATPMP +FUZZ_SUITE_LD_COMMON += $(NATPMP_LIBS) +endif + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -146,10 +156,16 @@ BITCOIN_TESTS += \ wallet/test/ismine_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp +FUZZ_SUITE_LD_COMMON +=\ + $(LIBBITCOIN_WALLET) \ + $(SQLITE_LIBS) \ + $(BDB_LIBS) + if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif + BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ @@ -173,12 +189,12 @@ test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $( if ENABLE_ZMQ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -if ENABLE_FUZZ - FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -279,8 +295,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/txrequest.cpp - -endif # ENABLE_FUZZ +endif # ENABLE_FUZZ_BINARY nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 4e858979fe..75fe68fcd1 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -16,6 +16,7 @@ libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAG libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ + test/fuzz/util.cpp \ $(TEST_FUZZ_H) LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) diff --git a/src/addrman.cpp b/src/addrman.cpp index ed7fccc0ff..f91121f156 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -9,6 +9,8 @@ #include <logging.h> #include <serialize.h> +#include <cmath> + int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); diff --git a/src/addrman.h b/src/addrman.h index 9ac67b7af6..92a5570953 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -335,22 +335,20 @@ public: * * nNew * * nTried * * number of "new" buckets XOR 2**30 - * * all nNew addrinfos in vvNew - * * all nTried addrinfos in vvTried - * * for each bucket: + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: * * number of elements - * * for each element: index + * * for each element: index in the serialized "all new addresses" + * * asmap checksum * * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it * as incompatible. This is necessary because it did not check the version number on * deserialization. * - * Notice that vvTried, mapAddr and vVector are never encoded explicitly; + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; * they are instead reconstructed from the other information. * - * vvNew is serialized, but only used if ADDRMAN_UNKNOWN_BUCKET_COUNT didn't change, - * otherwise it is reconstructed as well. - * * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * @@ -413,13 +411,13 @@ public: } } } - // Store asmap version after bucket entries so that it + // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_version; + uint256 asmap_checksum; if (m_asmap.size() != 0) { - asmap_version = SerializeHash(m_asmap); + asmap_checksum = SerializeHash(m_asmap); } - s << asmap_version; + s << asmap_checksum; } template <typename Stream> @@ -500,47 +498,63 @@ public: nTried -= nLost; // Store positions in the new table buckets to apply later (if possible). - std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing - - for (int bucket = 0; bucket < nUBuckets; bucket++) { - int nSize = 0; - s >> nSize; - for (int n = 0; n < nSize; n++) { - int nIndex = 0; - s >> nIndex; - if (nIndex >= 0 && nIndex < nNew) { - entryToBucket[nIndex] = bucket; + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector<std::pair<int, int>> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); } } } - uint256 supplied_asmap_version; + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum; if (m_asmap.size() != 0) { - supplied_asmap_version = SerializeHash(m_asmap); + supplied_asmap_checksum = SerializeHash(m_asmap); } - uint256 serialized_asmap_version; + uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_version; + s >> serialized_asmap_checksum; } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - int bucket = entryToBucket[n]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (format >= Format::V2_ASMAP && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && - info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + CAddrInfo& info = mapInfo[entry_index]; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } else { - // In case the new table data cannot be used (format unknown, bucket count wrong or new asmap), + // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); bucket = info.GetNewBucket(nKey, m_asmap); - nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][nUBucketPos] == -1) { - vvNew[bucket][nUBucketPos] = n; - info.nRefCount++; + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; } } } diff --git a/src/bench/bench.h b/src/bench/bench.h index bafc7f8716..22f06d8cb8 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <util/macros.h> + #include <chrono> #include <functional> #include <map> @@ -12,8 +14,6 @@ #include <vector> #include <bench/nanobench.h> -#include <boost/preprocessor/cat.hpp> -#include <boost/preprocessor/stringize.hpp> /* * Usage: @@ -56,8 +56,8 @@ public: static void RunAll(const Args& args); }; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo"); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index ffa772d8c1..d7b8c1badc 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -10,8 +10,6 @@ #include <random.h> #include <util/system.h> -#include <boost/thread/thread.hpp> - #include <vector> static const size_t BATCHES = 101; @@ -44,12 +42,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) void swap(PrevectorJob& x){p.swap(x.p);}; }; CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; - boost::thread_group tg; // The main thread should be counted to prevent thread oversubscription, and // to decrease the variance of benchmark results. - for (auto x = 0; x < GetNumCores() - 1; ++x) { - tg.create_thread([&]{queue.Thread();}); - } + queue.StartWorkerThreads(GetNumCores() - 1); // create all the data once, then submit copies in the benchmark. FastRandomContext insecure_rand(true); @@ -70,8 +65,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // it is done explicitly here for clarity control.Wait(); }); - tg.interrupt_all(); - tg.join_all(); + queue.StopWorkerThreads(); ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index e50c4476bb..3abfbfd784 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -42,20 +42,19 @@ static void CoinSelection(benchmark::Bench& bench) } addCoin(3 * COIN, wallet, wtxs); - // Create groups - std::vector<OutputGroup> groups; + // Create coins + std::vector<COutput> coins; for (const auto& wtx : wtxs) { - COutput output(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); - groups.emplace_back(output.GetInputCoin(), 6, false, 0, 0); + coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0); + const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0, false); bench.run([&] { std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool bnb_used; - bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used); + bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params, bnb_used); assert(success); assert(nValueRet == 1003 * COIN); assert(setCoinsRet.size() == 2); @@ -75,7 +74,8 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); - set.emplace_back(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0); + set.emplace_back(); + set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); wtxn.emplace_back(std::move(wtx)); } // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index dcd0e10285..f4fabedab6 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -76,7 +76,7 @@ static void PrevectorDeserialize(benchmark::Bench& bench) for (auto x = 0; x < 1000; ++x) { s0 >> t1; } - s0.Init(SER_NETWORK, 0); + s0.Rewind(); }); } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 4b45264a3c..45ed9f60dc 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -7,12 +7,15 @@ #include <rpc/blockchain.h> #include <streams.h> +#include <test/util/setup_common.h> #include <validation.h> #include <univalue.h> static void BlockToJsonVerbose(benchmark::Bench& bench) { + TestingSetup test_setup{}; + CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); char a = '\0'; stream.write(&a, 1); // Prevent compaction diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 94043a6b45..a9a0ec6f5d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -21,6 +21,7 @@ #include <util/url.h> #include <algorithm> +#include <cmath> #include <functional> #include <memory> #include <stdio.h> @@ -57,7 +58,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC getnewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -299,9 +300,12 @@ class NetinfoRequestHandler : public BaseRequestHandler { private: static constexpr int8_t UNKNOWN_NETWORK{-1}; - static constexpr uint8_t m_networks_size{3}; - const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion"}}; - std::array<std::array<uint16_t, m_networks_size + 2>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total/block-relay) + static constexpr int8_t NET_I2P{3}; // pos of "i2p" in m_networks + static constexpr uint8_t m_networks_size{4}; + const std::array<std::string, m_networks_size> m_networks{{"ipv4", "ipv6", "onion", "i2p"}}; + std::array<std::array<uint16_t, m_networks_size + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) + uint8_t m_block_relay_peers_count{0}; + uint8_t m_manual_peers_count{0}; int8_t NetworkStringToId(const std::string& str) const { for (uint8_t i = 0; i < m_networks_size; ++i) { @@ -315,12 +319,14 @@ private: bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } bool m_is_asmap_on{false}; + bool m_is_i2p_on{false}; size_t m_max_addr_length{0}; - size_t m_max_age_length{4}; + size_t m_max_age_length{3}; size_t m_max_id_length{2}; struct Peer { std::string addr; std::string sub_version; + std::string conn_type; std::string network; std::string age; double min_ping; @@ -332,6 +338,8 @@ private: int id; int mapped_as; int version; + bool is_bip152_hb_from; + bool is_bip152_hb_to; bool is_block_relay; bool is_outbound; bool operator<(const Peer& rhs) const { return std::tie(is_outbound, min_ping) < std::tie(rhs.is_outbound, rhs.min_ping); } @@ -350,6 +358,14 @@ private: const double milliseconds{round(1000 * seconds)}; return milliseconds > 999999 ? "-" : ToString(milliseconds); } + std::string ConnectionTypeForNetinfo(const std::string& conn_type) const + { + if (conn_type == "outbound-full-relay") return "full"; + if (conn_type == "block-relay-only") return "block"; + if (conn_type == "manual" || conn_type == "feeler") return conn_type; + if (conn_type == "addr-fetch") return "addr"; + return ""; + } const UniValue NetinfoHelp() { return std::string{ @@ -378,6 +394,9 @@ private: " type Type of peer connection\n" " \"full\" - full relay, the default\n" " \"block\" - block relay; like full relay but does not relay transactions or addresses\n" + " \"manual\" - peer we manually added using RPC addnode or the -addnode/-connect config options\n" + " \"feeler\" - short-lived connection for testing addresses\n" + " \"addr\" - address fetch; short-lived connection for requesting addresses\n" " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", or \"cjdns\")\n" " mping Minimum observed ping time, in milliseconds (ms)\n" " ping Last observed ping time, in milliseconds (ms)\n" @@ -385,6 +404,9 @@ private: " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" + " hb High-bandwidth BIP152 compact block relay\n" + " \".\" (to) - we selected the peer as a high-bandwidth peer\n" + " \"*\" (from) - the peer selected us as a high-bandwidth peer\n" " age Duration of connection to the peer, in minutes\n" " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" @@ -392,7 +414,7 @@ private: " address IP address and port of the peer\n" " version Peer version and subversion concatenated, e.g. \"70016/Satoshi:21.0.0/\"\n\n" "* The connection counts table displays the number of peers by direction, network, and the totals\n" - " for each, as well as a column for block relay peers.\n\n" + " for each, as well as two special outbound columns for block relay peers and manual peers.\n\n" "* The local addresses table lists each local address broadcast by the node, the port, and the score.\n\n" "Examples:\n\n" "Connection counts and local addresses only\n" @@ -449,16 +471,16 @@ public: const std::string network{peer["network"].get_str()}; const int8_t network_id{NetworkStringToId(network)}; if (network_id == UNKNOWN_NETWORK) continue; + m_is_i2p_on |= (network_id == NET_I2P); const bool is_outbound{!peer["inbound"].get_bool()}; const bool is_block_relay{!peer["relaytxes"].get_bool()}; + const std::string conn_type{peer["connection_type"].get_str()}; ++m_counts.at(is_outbound).at(network_id); // in/out by network ++m_counts.at(is_outbound).at(m_networks_size); // in/out overall ++m_counts.at(2).at(network_id); // total by network ++m_counts.at(2).at(m_networks_size); // total overall - if (is_block_relay) { - ++m_counts.at(is_outbound).at(m_networks_size + 1); // in/out block-relay - ++m_counts.at(2).at(m_networks_size + 1); // total block-relay - } + if (conn_type == "block-relay-only") ++m_block_relay_peers_count; + if (conn_type == "manual") ++m_manual_peers_count; if (DetailsRequested()) { // Push data for this peer to the peers vector. const int peer_id{peer["id"].get_int()}; @@ -474,7 +496,9 @@ public: const std::string addr{peer["addr"].get_str()}; const std::string age{conn_time == 0 ? "" : ToString((m_time_now - conn_time) / 60)}; const std::string sub_version{peer["subver"].get_str()}; - m_peers.push_back({addr, sub_version, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_block_relay, is_outbound}); + const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; + const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; + m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_bip152_hb_from, is_bip152_hb_to, is_block_relay, is_outbound}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); m_max_age_length = std::max(age.length(), m_max_age_length); m_max_id_length = std::max(ToString(peer_id).length(), m_max_id_length); @@ -488,15 +512,15 @@ public: // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("<-> relay net mping ping send recv txn blk %*s ", m_max_age_length, "age"); + result += strprintf("<-> type net mping ping send recv txn blk hb %*s ", m_max_age_length, "age"); if (m_is_asmap_on) result += " asmap "; result += strprintf("%*s %-*s%s\n", m_max_id_length, "id", IsAddressSelected() ? m_max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); for (const Peer& peer : m_peers) { std::string version{ToString(peer.version) + peer.sub_version}; result += strprintf( - "%3s %5s %5s%7s%7s%5s%5s%5s%5s %*s%*i %*s %-*s%s\n", + "%3s %6s %5s%7s%7s%5s%5s%5s%5s %2s %*s%*i %*s %-*s%s\n", peer.is_outbound ? "out" : "in", - peer.is_block_relay ? "block" : "full", + ConnectionTypeForNetinfo(peer.conn_type), peer.network, PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), @@ -504,6 +528,7 @@ public: peer.last_recv == 0 ? "" : ToString(m_time_now - peer.last_recv), peer.last_trxn == 0 ? "" : ToString((m_time_now - peer.last_trxn) / 60), peer.last_blck == 0 ? "" : ToString((m_time_now - peer.last_blck) / 60), + strprintf("%s%s", peer.is_bip152_hb_to ? "." : " ", peer.is_bip152_hb_from ? "*" : " "), m_max_age_length, // variable spacing peer.age, m_is_asmap_on ? 7 : 0, // variable spacing @@ -514,18 +539,27 @@ public: IsAddressSelected() ? peer.addr : "", IsVersionSelected() && version != "0" ? version : ""); } - result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); + result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); } // Report peer connection totals by type. - result += " ipv4 ipv6 onion total block-relay\n"; + result += " ipv4 ipv6 onion"; + if (m_is_i2p_on) result += " i2p"; + result += " total block"; + if (m_manual_peers_count) result += " manual"; const std::array<std::string, 3> rows{{"in", "out", "total"}}; - for (uint8_t i = 0; i < m_networks_size; ++i) { - result += strprintf("%-5s %5i %5i %5i %5i %5i\n", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2), m_counts.at(i).at(m_networks_size), m_counts.at(i).at(m_networks_size + 1)); + for (uint8_t i = 0; i < 3; ++i) { + result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts + if (m_is_i2p_on) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + result += strprintf(" %5i", m_counts.at(i).at(m_networks_size)); // total peers count + if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts + result += strprintf(" %5i", m_block_relay_peers_count); + if (m_manual_peers_count) result += strprintf(" %5i", m_manual_peers_count); + } } // Report local addresses, ports, and scores. - result += "\nLocal addresses"; + result += "\n\nLocal addresses"; const std::vector<UniValue>& local_addrs{networkinfo["localaddresses"].getValues()}; if (local_addrs.empty()) { result += ": n/a\n"; diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index b702a68bdf..af07b28d3d 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -25,6 +25,7 @@ #include <util/system.h> #include <util/translation.h> +#include <atomic> #include <functional> #include <memory> #include <stdio.h> @@ -181,7 +182,16 @@ static int CommandLineUtil(int argc, char* argv[]) return nRet; } +#ifdef WIN32 +// Export main() and ensure working ASLR on Windows. +// Exporting a symbol will prevent the linker from stripping +// the .reloc section from the binary, which is a requirement +// for ASLR. This is a temporary workaround until a fixed +// version of binutils is used for releases. +__declspec(dllexport) int main(int argc, char* argv[]) +#else int main(int argc, char* argv[]) +#endif { SetupEnvironment(); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 3e8e5fc7bc..b84d909b07 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -33,51 +33,52 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddCommand("info", "Get wallet info", OptionsCategory::COMMANDS); + argsman.AddCommand("create", "Create new wallet file", OptionsCategory::COMMANDS); + argsman.AddCommand("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", OptionsCategory::COMMANDS); + argsman.AddCommand("dump", "Print out all of the wallet key-value records", OptionsCategory::COMMANDS); + argsman.AddCommand("createfromdump", "Create new wallet file from dumped records", OptionsCategory::COMMANDS); } -static bool WalletAppInit(int argc, char* argv[]) +static bool WalletAppInit(ArgsManager& args, int argc, char* argv[]) { - SetupWalletToolArgs(gArgs); + SetupWalletToolArgs(args); std::string error_message; - if (!gArgs.ParseParameters(argc, argv, error_message)) { + if (!args.ParseParameters(argc, argv, error_message)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (argc < 2 || HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; - if (!gArgs.IsArgSet("-version")) { - strUsage += "\n" - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" - "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" - "Usage:\n" - " bitcoin-wallet [options] <command>\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); - } + if (!args.IsArgSet("-version")) { + strUsage += "\n" + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n"; + strUsage += "\n" + args.GetHelpMessage(); + } tfm::format(std::cout, "%s", strUsage); return false; } // check for printtoconsole, allow -debug - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", args.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")); return false; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); return true; } int main(int argc, char* argv[]) { + ArgsManager& args = gArgs; #ifdef WIN32 util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); @@ -85,7 +86,7 @@ int main(int argc, char* argv[]) SetupEnvironment(); RandomInit(); try { - if (!WalletAppInit(argc, argv)) return EXIT_FAILURE; + if (!WalletAppInit(args, argc, argv)) return EXIT_FAILURE; } catch (const std::exception& e) { PrintExceptionContinue(&e, "WalletAppInit()"); return EXIT_FAILURE; @@ -94,33 +95,19 @@ int main(int argc, char* argv[]) return EXIT_FAILURE; } - std::string method {}; - for(int i = 1; i < argc; ++i) { - if (!IsSwitchChar(argv[i][0])) { - if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); - return EXIT_FAILURE; - } - method = argv[i]; - } - } - - if (method.empty()) { + const auto command = args.GetCommand(); + if (!command) { tfm::format(std::cerr, "No method provided. Run `bitcoin-wallet -help` for valid methods.\n"); return EXIT_FAILURE; } - - // A name must be provided when creating a file - if (method == "create" && !gArgs.IsArgSet("-wallet")) { - tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + if (command->args.size() != 0) { + tfm::format(std::cerr, "Error: Additional arguments provided (%s). Methods do not take arguments. Please refer to `-help`.\n", Join(command->args, ", ")); return EXIT_FAILURE; } - std::string name = gArgs.GetArg("-wallet", ""); - ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) { + if (!WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; } ECC_Stop(); diff --git a/src/checkqueue.h b/src/checkqueue.h index e3faa1dec0..4ceeb3600a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -6,13 +6,12 @@ #define BITCOIN_CHECKQUEUE_H #include <sync.h> +#include <tinyformat.h> +#include <util/threadnames.h> #include <algorithm> #include <vector> -#include <boost/thread/condition_variable.hpp> -#include <boost/thread/mutex.hpp> - template <typename T> class CCheckQueueControl; @@ -31,61 +30,64 @@ class CCheckQueue { private: //! Mutex to protect the inner state - boost::mutex mutex; + Mutex m_mutex; //! Worker threads block on this when out of work - boost::condition_variable condWorker; + std::condition_variable m_worker_cv; //! Master thread blocks on this when out of work - boost::condition_variable condMaster; + std::condition_variable m_master_cv; //! The queue of elements to be processed. //! As the order of booleans doesn't matter, it is used as a LIFO (stack) - std::vector<T> queue; + std::vector<T> queue GUARDED_BY(m_mutex); //! The number of workers (including the master) that are idle. - int nIdle; + int nIdle GUARDED_BY(m_mutex){0}; //! The total number of workers (including the master). - int nTotal; + int nTotal GUARDED_BY(m_mutex){0}; //! The temporary evaluation result. - bool fAllOk; + bool fAllOk GUARDED_BY(m_mutex){true}; /** * Number of verifications that haven't completed yet. * This includes elements that are no longer queued, but still in the * worker's own batches. */ - unsigned int nTodo; + unsigned int nTodo GUARDED_BY(m_mutex){0}; //! The maximum number of elements to be processed in one batch - unsigned int nBatchSize; + const unsigned int nBatchSize; + + std::vector<std::thread> m_worker_threads; + bool m_request_stop GUARDED_BY(m_mutex){false}; /** Internal function that does bulk of the verification work. */ - bool Loop(bool fMaster = false) + bool Loop(bool fMaster) { - boost::condition_variable& cond = fMaster ? condMaster : condWorker; + std::condition_variable& cond = fMaster ? m_master_cv : m_worker_cv; std::vector<T> vChecks; vChecks.reserve(nBatchSize); unsigned int nNow = 0; bool fOk = true; do { { - boost::unique_lock<boost::mutex> lock(mutex); + WAIT_LOCK(m_mutex, lock); // first do the clean-up of the previous loop run (allowing us to do it in the same critsect) if (nNow) { fAllOk &= fOk; nTodo -= nNow; if (nTodo == 0 && !fMaster) // We processed the last element; inform the master it can exit and return the result - condMaster.notify_one(); + m_master_cv.notify_one(); } else { // first iteration nTotal++; } // logically, the do loop starts here - while (queue.empty()) { + while (queue.empty() && !m_request_stop) { if (fMaster && nTodo == 0) { nTotal--; bool fRet = fAllOk; @@ -98,6 +100,10 @@ private: cond.wait(lock); // wait nIdle--; } + if (m_request_stop) { + return false; + } + // Decide how many work units to process now. // * Do not try to do everything at once, but aim for increasingly smaller batches so // all workers finish approximately simultaneously. @@ -106,7 +112,7 @@ private: nNow = std::max(1U, std::min(nBatchSize, (unsigned int)queue.size() / (nTotal + nIdle + 1))); vChecks.resize(nNow); for (unsigned int i = 0; i < nNow; i++) { - // We want the lock on the mutex to be as short as possible, so swap jobs from the global + // We want the lock on the m_mutex to be as short as possible, so swap jobs from the global // queue to the local batch vector instead of copying. vChecks[i].swap(queue.back()); queue.pop_back(); @@ -124,40 +130,68 @@ private: public: //! Mutex to ensure only one concurrent CCheckQueueControl - boost::mutex ControlMutex; + Mutex m_control_mutex; //! Create a new check queue - explicit CCheckQueue(unsigned int nBatchSizeIn) : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), nBatchSize(nBatchSizeIn) {} + explicit CCheckQueue(unsigned int nBatchSizeIn) + : nBatchSize(nBatchSizeIn) + { + } - //! Worker thread - void Thread() + //! Create a pool of new worker threads. + void StartWorkerThreads(const int threads_num) { - Loop(); + { + LOCK(m_mutex); + nIdle = 0; + nTotal = 0; + fAllOk = true; + } + assert(m_worker_threads.empty()); + for (int n = 0; n < threads_num; ++n) { + m_worker_threads.emplace_back([this, n]() { + util::ThreadRename(strprintf("scriptch.%i", n)); + Loop(false /* worker thread */); + }); + } } //! Wait until execution finishes, and return whether all evaluations were successful. bool Wait() { - return Loop(true); + return Loop(true /* master thread */); } //! Add a batch of checks to the queue void Add(std::vector<T>& vChecks) { - boost::unique_lock<boost::mutex> lock(mutex); + LOCK(m_mutex); for (T& check : vChecks) { queue.push_back(T()); check.swap(queue.back()); } nTodo += vChecks.size(); if (vChecks.size() == 1) - condWorker.notify_one(); + m_worker_cv.notify_one(); else if (vChecks.size() > 1) - condWorker.notify_all(); + m_worker_cv.notify_all(); + } + + //! Stop all of the worker threads. + void StopWorkerThreads() + { + WITH_LOCK(m_mutex, m_request_stop = true); + m_worker_cv.notify_all(); + for (std::thread& t : m_worker_threads) { + t.join(); + } + m_worker_threads.clear(); + WITH_LOCK(m_mutex, m_request_stop = false); } ~CCheckQueue() { + assert(m_worker_threads.empty()); } }; @@ -181,7 +215,7 @@ public: { // passed queue is supposed to be unused, or nullptr if (pqueue != nullptr) { - ENTER_CRITICAL_SECTION(pqueue->ControlMutex); + ENTER_CRITICAL_SECTION(pqueue->m_control_mutex); } } @@ -205,7 +239,7 @@ public: if (!fDone) Wait(); if (pqueue != nullptr) { - LEAVE_CRITICAL_SECTION(pqueue->ControlMutex); + LEAVE_CRITICAL_SECTION(pqueue->m_control_mutex); } } }; diff --git a/src/clientversion.h b/src/clientversion.h index 2da909f829..0ed3f68094 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CLIENTVERSION_H #define BITCOIN_CLIENTVERSION_H +#include <util/macros.h> + #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H @@ -14,13 +16,6 @@ #error Client version information missing: version is not defined by bitcoin-config.h or in any other way #endif -/** - * Converts the parameter X to a string after macro replacement on X has been performed. - * Don't merge these into one macro! - */ -#define STRINGIZE(X) DO_STRINGIZE(X) -#define DO_STRINGIZE(X) #X - //! Copyright string used in Windows .rc files #define COPYRIGHT_STR "2009-" STRINGIZE(COPYRIGHT_YEAR) " " COPYRIGHT_HOLDERS_FINAL diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 6d7a293f9b..8a51f310f7 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -54,6 +54,12 @@ __asm(".symver log2f_old,log2f@GLIBC_2.2.5"); __asm(".symver log2f_old,log2f@GLIBC_2.4"); #elif defined(__aarch64__) __asm(".symver log2f_old,log2f@GLIBC_2.17"); +#elif defined(__powerpc64__) +# ifdef WORDS_BIGENDIAN +__asm(".symver log2f_old,log2f@GLIBC_2.3"); +# else +__asm(".symver log2f_old,log2f@GLIBC_2.17"); +# endif #elif defined(__riscv) __asm(".symver log2f_old,log2f@GLIBC_2.27"); #endif diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 33e4b366a1..c119036db2 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -8,9 +8,10 @@ #include <clientversion.h> #include <fs.h> #include <serialize.h> +#include <span.h> #include <streams.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> @@ -73,12 +74,12 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue(ssValue.data(), ssValue.size()); + leveldb::Slice slValue((const char*)ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); // LevelDB serializes writes as: @@ -98,7 +99,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); batch.Delete(slKey); // LevelDB serializes erases as: @@ -137,7 +138,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); piter->Seek(slKey); } @@ -146,7 +147,7 @@ public: template<typename K> bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); try { - CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssKey(MakeUCharSpan(slKey), SER_DISK, CLIENT_VERSION); ssKey >> key; } catch (const std::exception&) { return false; @@ -157,7 +158,7 @@ public: template<typename V> bool GetValue(V& value) { leveldb::Slice slValue = piter->value(); try { - CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(slValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { @@ -232,7 +233,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -243,7 +244,7 @@ public: dbwrapper_private::HandleError(status); } try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(strValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { @@ -266,7 +267,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -310,8 +311,8 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); pdb->GetApproximateSizes(&range, 1, &size); @@ -329,11 +330,10 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); pdb->CompactRange(&slKey1, &slKey2); } - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/fs.cpp b/src/fs.cpp index 7eb0f5ca9f..4f20ca4d28 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -31,6 +31,12 @@ FILE *fopen(const fs::path& p, const char *mode) #endif } +fs::path AbsPathJoin(const fs::path& base, const fs::path& path) +{ + assert(base.is_absolute()); + return fs::absolute(path, base); +} + #ifndef WIN32 static std::string GetErrorReason() @@ -21,6 +21,17 @@ namespace fs = boost::filesystem; namespace fsbridge { FILE *fopen(const fs::path& p, const char *mode); + /** + * Helper function for joining two paths + * + * @param[in] base Base path + * @param[in] path Path to combine with base + * @returns path unchanged if it is an absolute path, otherwise returns base joined with path. Returns base unchanged if path is empty. + * @pre Base path must be absolute + * @post Returned path will always be absolute + */ + fs::path AbsPathJoin(const fs::path& base, const fs::path& path); + class FileLock { public: diff --git a/src/index/base.cpp b/src/index/base.cpp index e67b813763..3d3dda95b1 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -62,7 +62,7 @@ bool BaseIndex::Init() if (locator.IsNull()) { m_best_block_index = nullptr; } else { - m_best_block_index = FindForkInGlobalIndex(::ChainActive(), locator); + m_best_block_index = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); } m_synced = m_best_block_index.load() == ::ChainActive().Tip(); return true; @@ -239,7 +239,7 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) const CBlockIndex* locator_tip_index; { LOCK(cs_main); - locator_tip_index = LookupBlockIndex(locator_tip_hash); + locator_tip_index = g_chainman.m_blockman.LookupBlockIndex(locator_tip_hash); } if (!locator_tip_index) { diff --git a/src/init.cpp b/src/init.cpp index 09eb76eaee..a77e2cf1cb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -68,6 +68,8 @@ #include <set> #include <stdint.h> #include <stdio.h> +#include <thread> +#include <vector> #ifndef WIN32 #include <attributes.h> @@ -78,7 +80,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/signals2/signal.hpp> -#include <boost/thread/thread.hpp> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -155,8 +156,6 @@ static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; static std::thread g_load_block; -static boost::thread_group threadGroup; - void Interrupt(NodeContext& node) { InterruptHTTPServer(); @@ -218,11 +217,10 @@ void Shutdown(NodeContext& node) StopTorControl(); // After everything has been shut down, but before things get flushed, stop the - // CScheduler/checkqueue, threadGroup and load block thread. + // CScheduler/checkqueue, scheduler and load block thread. if (node.scheduler) node.scheduler->stop(); if (g_load_block.joinable()) g_load_block.join(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. @@ -447,7 +445,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks. Warning: if it is used with ipv4 or ipv6 but not onion and the -onion or -proxy option is set, then outbound onion connections will still be made; use -noonion or -onion=0 to disable outbound onion connections in this case.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -456,8 +454,8 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP @@ -521,10 +519,11 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); #ifdef HAVE_THREAD_LOCAL @@ -704,7 +703,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp if (!file) break; // This error is logged in OpenBlockFile LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - LoadExternalBlockFile(chainparams, file, &pos); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file, &pos); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -723,7 +722,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); - LoadExternalBlockFile(chainparams, file); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -1042,16 +1041,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Trim requested connection counts, to fit into system limitations // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 - nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind); + nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); + #ifdef USE_POLL int fd_max = nFD; #else int fd_max = FD_SETSIZE; #endif - nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); + nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); - nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); + nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE, nMaxConnections); if (nMaxConnections < nUserMaxConnections) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); @@ -1334,16 +1334,14 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA LogPrintf("Script verification uses %d additional threads\n", script_threads); if (script_threads >= 1) { g_parallel_script_checks = true; - for (int i = 0; i < script_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_threads); } assert(!node.scheduler); node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); + node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1612,7 +1610,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). if (!chainman.BlockIndex().empty() && - !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { + !g_chainman.m_blockman.LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } diff --git a/src/key_io.cpp b/src/key_io.cpp index a270ede864..e27673fd16 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -12,6 +12,9 @@ #include <assert.h> #include <string.h> +/// Maximum witness length for Bech32 addresses. +static constexpr std::size_t BECH32_WITNESS_PROG_MAX_LEN = 40; + namespace { class DestinationEncoder { @@ -65,10 +68,11 @@ public: std::string operator()(const CNoDestination& no) const { return {}; } }; -CTxDestination DecodeDestination(const std::string& str, const CChainParams& params) +CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str) { std::vector<unsigned char> data; uint160 hash; + error_str = ""; if (DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). @@ -85,10 +89,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin() + script_prefix.size(), data.end(), hash.begin()); return ScriptHash(hash); } + + // Set potential error message. + // This message may be changed if the address can also be interpreted as a Bech32 address. + error_str = "Invalid prefix for Base58-encoded address"; } data.clear(); auto bech = bech32::Decode(str); - if (bech.second.size() > 0 && bech.first == params.Bech32HRP()) { + if (bech.second.size() > 0) { + error_str = ""; + + if (bech.first != params.Bech32HRP()) { + error_str = "Invalid prefix for Bech32 address"; + return CNoDestination(); + } + // Bech32 decoding int version = bech.second[0]; // The first 5 bit symbol is the witness version (0-16) // The rest of the symbols are converted witness program bytes. @@ -109,11 +124,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return scriptid; } } + + error_str = "Invalid Bech32 v0 address data size"; + return CNoDestination(); + } + + if (version > 16) { + error_str = "Invalid Bech32 address witness version"; return CNoDestination(); } - if (version > 16 || data.size() < 2 || data.size() > 40) { + + if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { + error_str = "Invalid Bech32 address data size"; return CNoDestination(); } + WitnessUnknown unk; unk.version = version; std::copy(data.begin(), data.end(), unk.program); @@ -121,6 +146,10 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return unk; } } + + // Set error message if address can't be interpreted as Base58 or Bech32. + if (error_str.empty()) error_str = "Invalid address format"; + return CNoDestination(); } } // namespace @@ -208,14 +237,21 @@ std::string EncodeDestination(const CTxDestination& dest) return std::visit(DestinationEncoder(Params()), dest); } +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg) +{ + return DecodeDestination(str, Params(), error_msg); +} + CTxDestination DecodeDestination(const std::string& str) { - return DecodeDestination(str, Params()); + std::string error_msg; + return DecodeDestination(str, error_msg); } bool IsValidDestinationString(const std::string& str, const CChainParams& params) { - return IsValidDestination(DecodeDestination(str, params)); + std::string error_msg; + return IsValidDestination(DecodeDestination(str, params, error_msg)); } bool IsValidDestinationString(const std::string& str) diff --git a/src/key_io.h b/src/key_io.h index d80c08f49c..bd81f7847e 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -23,6 +23,7 @@ std::string EncodeExtPubKey(const CExtPubKey& extpubkey); std::string EncodeDestination(const CTxDestination& dest); CTxDestination DecodeDestination(const std::string& str); +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg); bool IsValidDestinationString(const std::string& str); bool IsValidDestinationString(const std::string& str, const CChainParams& params); diff --git a/src/miner.cpp b/src/miner.cpp index 009ae6b13a..076d43c951 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,7 @@ void RegenerateCommitments(CBlock& block) tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); block.vtx.at(0) = MakeTransactionRef(tx); - GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); block.hashMerkleRoot = BlockMerkleRoot(block); } @@ -176,7 +176,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } int64_t nTime2 = GetTimeMicros(); diff --git a/src/net.cpp b/src/net.cpp index 16aa489873..5fa405a690 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -36,6 +36,7 @@ #include <algorithm> #include <cstdint> +#include <functional> #include <unordered_map> #include <math.h> @@ -817,7 +818,7 @@ size_t CConnman::SocketSendData(CNode& node) const // error int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { - LogPrintf("socket send error %s\n", NetworkErrorString(nErr)); + LogPrint(BCLog::NET, "socket send error for peer=%d: %s\n", node.GetId(), NetworkErrorString(nErr)); node.CloseSocketDisconnect(); } } @@ -1006,6 +1007,7 @@ bool CConnman::AttemptToEvictConnection() LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (pnode->GetId() == *node_id_to_evict) { + LogPrint(BCLog::NET, "selected %s connection for eviction peer=%d; disconnecting\n", pnode->ConnectionTypeAsString(), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -1054,7 +1056,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } if (!fNetworkActive) { - LogPrintf("connection from %s dropped: not accepting new connections\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s dropped: not accepting new connections\n", addr.ToString()); CloseSocket(hSocket); return; } @@ -1219,37 +1221,47 @@ void CConnman::NotifyNumConnectionsChanged() } } -void CConnman::InactivityCheck(CNode *pnode) const +bool CConnman::InactivityCheck(const CNode& node) const { - int64_t nTime = GetSystemTimeInSeconds(); - if (nTime - pnode->nTimeConnected > m_peer_connect_timeout) - { - if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) - { - LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d from %d\n", m_peer_connect_timeout, pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); - pnode->fDisconnect = true; - } - else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) - { - LogPrintf("socket sending timeout: %is\n", nTime - pnode->nLastSend); - pnode->fDisconnect = true; - } - else if (nTime - pnode->nLastRecv > TIMEOUT_INTERVAL) - { - LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); - pnode->fDisconnect = true; - } - else if (pnode->nPingNonceSent && pnode->m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) - { - LogPrintf("ping timeout: %fs\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - pnode->m_ping_start.load())); - pnode->fDisconnect = true; - } - else if (!pnode->fSuccessfullyConnected) - { - LogPrint(BCLog::NET, "version handshake timeout from %d\n", pnode->GetId()); - pnode->fDisconnect = true; - } + // Use non-mockable system time (otherwise these timers will pop when we + // use setmocktime in the tests). + int64_t now = GetSystemTimeInSeconds(); + + if (now <= node.nTimeConnected + m_peer_connect_timeout) { + // Only run inactivity checks if the peer has been connected longer + // than m_peer_connect_timeout. + return false; + } + + if (node.nLastRecv == 0 || node.nLastSend == 0) { + LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); + return true; + } + + if (now > node.nLastSend + TIMEOUT_INTERVAL) { + LogPrint(BCLog::NET, "socket sending timeout: %is peer=%d\n", now - node.nLastSend, node.GetId()); + return true; + } + + if (now > node.nLastRecv + TIMEOUT_INTERVAL) { + LogPrint(BCLog::NET, "socket receive timeout: %is peer=%d\n", now - node.nLastRecv, node.GetId()); + return true; + } + + if (node.nPingNonceSent && node.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) { + // We use mockable time for ping timeouts. This means that setmocktime + // may cause pings to time out for peers that have been connected for + // longer than m_peer_connect_timeout. + LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - node.m_ping_start.load()), node.GetId()); + return true; + } + + if (!node.fSuccessfullyConnected) { + LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId()); + return true; } + + return false; } bool CConnman::GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set) @@ -1525,7 +1537,7 @@ void CConnman::SocketHandler() if (bytes_sent) RecordBytesSent(bytes_sent); } - InactivityCheck(pnode); + if (InactivityCheck(*pnode)) pnode->fDisconnect = true; } { LOCK(cs_vNodes); @@ -2682,6 +2694,7 @@ bool CConnman::DisconnectNode(const std::string& strNode) { LOCK(cs_vNodes); if (CNode* pnode = FindNode(strNode)) { + LogPrint(BCLog::NET, "disconnect by address%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2694,6 +2707,7 @@ bool CConnman::DisconnectNode(const CSubNet& subnet) LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (subnet.Match(pnode->addr)) { + LogPrint(BCLog::NET, "disconnect by subnet%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->GetId()); pnode->fDisconnect = true; disconnected = true; } @@ -2711,6 +2725,7 @@ bool CConnman::DisconnectNode(NodeId id) LOCK(cs_vNodes); for(CNode* pnode : vNodes) { if (id == pnode->GetId()) { + LogPrint(BCLog::NET, "disconnect by id peer=%d; disconnecting\n", pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2864,6 +2879,9 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + } // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; @@ -2879,18 +2897,14 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; - if (pnode->nSendSize > nSendBufferMaxSize) - pnode->fPauseSend = true; + if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) - pnode->vSendMsg.push_back(std::move(msg.data)); + if (nMessageSize) pnode->vSendMsg.push_back(std::move(msg.data)); // If write queue empty, attempt "optimistic write" - if (optimisticSend == true) - nBytesSent = SocketSendData(*pnode); + if (optimisticSend) nBytesSent = SocketSendData(*pnode); } - if (nBytesSent) - RecordBytesSent(nBytesSent); + if (nBytesSent) RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -2933,3 +2947,31 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } + +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming) +{ + // Note: This function captures the message at the time of processing, + // not at socket receive/send time. + // This ensures that the messages are always in order from an application + // layer (processing) perspective. + auto now = GetTime<std::chrono::microseconds>(); + + // Windows folder names can not include a colon + std::string clean_addr = addr.ToString(); + std::replace(clean_addr.begin(), clean_addr.end(), ':', '_'); + + fs::path base_path = GetDataDir() / "message_capture" / clean_addr; + fs::create_directories(base_path); + + fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); + CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + + ser_writedata64(f, now.count()); + f.write(msg_type.data(), msg_type.length()); + for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { + f << '\0'; + } + uint32_t size = data.size(); + ser_writedata32(f, size); + f.write((const char*)data.data(), data.size()); +} @@ -20,6 +20,7 @@ #include <policy/feerate.h> #include <protocol.h> #include <random.h> +#include <span.h> #include <streams.h> #include <sync.h> #include <threadinterrupt.h> @@ -75,6 +76,8 @@ static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; +/** Number of file descriptors required for message capture **/ +static const int NUM_FDS_MESSAGE_CAPTURE = 1; static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; @@ -111,7 +114,8 @@ struct CSerializedNetMsg * connection. Aside from INBOUND, all types are initiated by us. * * If adding or removing types, please update CONNECTION_TYPE_DOC in - * src/rpc/net.cpp. */ + * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in + * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ enum class ConnectionType { /** * Inbound connections are those initiated by a peer. This is the only @@ -122,7 +126,7 @@ enum class ConnectionType { /** * These are the default connections that we use to connect with the - * network. There is no restriction on what is relayed- by default we relay + * network. There is no restriction on what is relayed; by default we relay * blocks, addresses & transactions. We automatically attempt to open * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. */ @@ -130,8 +134,8 @@ enum class ConnectionType { /** - * We open manual connections to addresses that users explicitly inputted - * via the addnode RPC, or the -connect command line argument. Even if a + * We open manual connections to addresses that users explicitly requested + * via the addnode RPC or the -addnode/-connect configuration options. Even if a * manual connection is misbehaving, we do not automatically disconnect or * add it to our discouragement filter. */ @@ -150,7 +154,7 @@ enum class ConnectionType { * although in our codebase feeler connections encompass test-before-evict as well. * We make these connections approximately every FEELER_INTERVAL: * first we resolve previously found collisions if they exist (test-before-evict), - * otherwise connect to a node from the new table. + * otherwise we connect to a node from the new table. */ FEELER, @@ -441,6 +445,7 @@ public: * messages, implying a preference to receive ADDRv2 instead of ADDR ones. */ std::atomic_bool m_wants_addrv2{false}; + /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the // next time DisconnectNodes() runs @@ -1044,7 +1049,8 @@ private: void AcceptConnection(const ListenSocket& hListenSocket); void DisconnectNodes(); void NotifyNumConnectionsChanged(); - void InactivityCheck(CNode *pnode) const; + /** Return true if the peer is inactive and should be disconnected. */ + bool InactivityCheck(const CNode& node) const; bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); void SocketHandler(); @@ -1239,6 +1245,9 @@ inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; } +/** Dump binary message to file, with timestamp */ +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming); + struct NodeEvictionCandidate { NodeId id; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cf73b1dae2..6ee9846c52 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -676,7 +676,7 @@ static void ProcessBlockAvailability(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_ assert(state != nullptr); if (!state->hashLastUnknownBlock.IsNull()) { - const CBlockIndex* pindex = LookupBlockIndex(state->hashLastUnknownBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(state->hashLastUnknownBlock); if (pindex && pindex->nChainWork > 0) { if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { state->pindexBestKnownBlock = pindex; @@ -693,7 +693,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV ProcessBlockAvailability(nodeid); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex && pindex->nChainWork > 0) { // An actually better block was announced. if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { @@ -880,13 +880,14 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, int64_t nTime) CAddress(CService(), addr.nServices); CAddress addrMe = CAddress(CService(), nLocalNodeServices); + const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr; m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr)); + nonce, strSubVersion, nNodeStartingHeight, tx_relay)); if (fLogIPs) { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), tx_relay, nodeid); } else { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), tx_relay, nodeid); } } @@ -1595,7 +1596,7 @@ bool static AlreadyHaveTx(const GenTxid& gtxid, const CTxMemPool& mempool) EXCLU bool static AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - return LookupBlockIndex(block_hash) != nullptr; + return g_chainman.m_blockman.LookupBlockIndex(block_hash) != nullptr; } void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) @@ -1685,7 +1686,7 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch bool need_activate_chain = false; { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { if (pindex->HaveTxsDownloaded() && !pindex->IsValid(BLOCK_VALID_SCRIPTS) && pindex->IsValid(BLOCK_VALID_TREE)) { @@ -1700,13 +1701,13 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch } // release cs_main before calling ActivateBestChain if (need_activate_chain) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { send = BlockRequestAllowed(pindex, consensusParams); if (!send) { @@ -1996,7 +1997,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // don't connect before giving DoS points // - Once a headers message is received that is valid and does connect, // nUnconnectingHeaders gets reset back to 0. - if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + if (!g_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", @@ -2026,7 +2027,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // If we don't have the last header, then they'll have given us // something new (if these headers are valid). - if (!LookupBlockIndex(hashLastBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { received_new_header = true; } } @@ -2278,7 +2279,7 @@ static bool PrepareBlockFilterRequest(CNode& peer, const CChainParams& chain_par { LOCK(cs_main); - stop_index = LookupBlockIndex(stop_hash); + stop_index = g_chainman.m_blockman.LookupBlockIndex(stop_hash); // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { @@ -2488,6 +2489,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool fRelay = true; vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; + if (nTime < 0) { + nTime = 0; + } nServices = ServiceFlags(nServiceInt); if (!pfrom.IsInboundConn()) { @@ -2642,9 +2646,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fLogIPs) remoteAddr = ", peeraddr=" + pfrom.addr.ToString(); - LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d%s\n", cleanSubVer, pfrom.nVersion, - peer->m_starting_height, addrMe.ToString(), pfrom.GetId(), + peer->m_starting_height, addrMe.ToString(), fRelay, pfrom.GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -2659,6 +2663,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Feeler connections exist only to verify if address is online. if (pfrom.IsFeelerConn()) { + LogPrint(BCLog::NET, "feeler connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; } return; @@ -2674,7 +2679,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); if (msg_type == NetMsgType::VERACK) { - if (pfrom.fSuccessfullyConnected) return; + if (pfrom.fSuccessfullyConnected) { + LogPrint(BCLog::NET, "ignoring redundant verack message from peer=%d\n", pfrom.GetId()); + return; + } if (!pfrom.IsInboundConn()) { LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s (%s)\n", @@ -2740,12 +2748,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // Feature negotiation of wtxidrelay must happen between VERSION and VERACK - // to avoid relay problems from switching after a connection is up. + // BIP339 defines feature negotiation of wtxidrelay, which must happen between + // VERSION and VERACK to avoid relay problems from switching after a connection is up. if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send wtxidrelay message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a wtxidrelay message after VERACK. + LogPrint(BCLog::NET, "wtxidrelay received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2754,15 +2762,21 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!State(pfrom.GetId())->m_wtxid_relay) { State(pfrom.GetId())->m_wtxid_relay = true; g_wtxid_relay_peers++; + } else { + LogPrint(BCLog::NET, "ignoring duplicate wtxidrelay from peer=%d\n", pfrom.GetId()); } + } else { + LogPrint(BCLog::NET, "ignoring wtxidrelay due to old common version=%d from peer=%d\n", pfrom.GetCommonVersion(), pfrom.GetId()); } return; } + // BIP155 defines feature negotiation of addrv2 and sendaddrv2, which must happen + // between VERSION and VERACK. if (msg_type == NetMsgType::SENDADDRV2) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send SENDADDRV2 message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a SENDADDRV2 message after VERACK. + LogPrint(BCLog::NET, "sendaddrv2 received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2789,6 +2803,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, s >> vAddr; if (!pfrom.RelayAddrsWithConn()) { + LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); return; } if (vAddr.size() > MAX_ADDR_TO_SEND) @@ -2832,8 +2847,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; - if (pfrom.IsAddrFetchConn()) + if (pfrom.IsAddrFetchConn()) { + LogPrint(BCLog::NET, "addrfetch connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; + } return; } @@ -2960,7 +2977,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, a_recent_block = most_recent_block; } BlockValidationState state; - if (!ActivateBestChain(state, m_chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, m_chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -2968,7 +2985,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LOCK(cs_main); // Find the last block the caller has in the main chain - const CBlockIndex* pindex = FindForkInGlobalIndex(::ChainActive(), locator); + const CBlockIndex* pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); // Send the rest of the chain if (pindex) @@ -3021,7 +3038,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(req.blockhash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(req.blockhash); if (!pindex || !(pindex->nStatus & BLOCK_HAVE_DATA)) { LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom.GetId()); return; @@ -3075,7 +3092,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (locator.IsNull()) { // If locator is null, return the hashStop block - pindex = LookupBlockIndex(hashStop); + pindex = g_chainman.m_blockman.LookupBlockIndex(hashStop); if (!pindex) { return; } @@ -3088,7 +3105,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, else { // Find the last block the caller has in the main chain - pindex = FindForkInGlobalIndex(::ChainActive(), locator); + pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); if (pindex) pindex = ::ChainActive().Next(pindex); } @@ -3352,14 +3369,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } - if (!LookupBlockIndex(cmpctblock.header.GetHash())) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.GetHash())) { received_new_header = true; } } @@ -3840,6 +3857,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERLOAD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3862,6 +3880,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERADD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3889,6 +3908,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERCLEAR) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -4028,14 +4048,12 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + if (pfrom->fPauseSend) return false; std::list<CNetMessage> msgs; { LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; + if (pfrom->vProcessMsg.empty()) return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; @@ -4044,6 +4062,10 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } CNetMessage& msg(msgs.front()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + } + msg.SetVersion(pfrom->GetCommonVersion()); const std::string& msg_type = msg.m_command; @@ -4425,7 +4447,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // then send all headers past that one. If we come across any // headers that aren't on ::ChainActive(), give up. for (const uint256& hash : peer->m_blocks_for_headers_relay) { - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); assert(pindex); if (::ChainActive()[pindex->nHeight] != pindex) { // Bail out if we reorged away from this block @@ -4517,7 +4539,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // in the past. if (!peer->m_blocks_for_headers_relay.empty()) { const uint256& hashToAnnounce = peer->m_blocks_for_headers_relay.back(); - const CBlockIndex* pindex = LookupBlockIndex(hashToAnnounce); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashToAnnounce); assert(pindex); // Warn if we're announcing a block that is not on the main chain. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 02e50c4dbe..56113cb731 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -63,7 +63,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + stats.nHeight = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock)->nHeight; } PrepareHash(hash_obj, stats); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index e07eaa33d8..b7efd68cfd 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -447,7 +447,7 @@ public: { LOCK(cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (CBlockIndex* fork = FindForkInGlobalIndex(active, locator)) { + if (CBlockIndex* fork = g_chainman.m_blockman.FindForkInGlobalIndex(active, locator)) { return fork->nHeight; } return nullopt; @@ -456,7 +456,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - return FillBlock(LookupBlockIndex(hash), block, lock, active); + return FillBlock(g_chainman.m_blockman.LookupBlockIndex(hash), block, lock, active); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { @@ -468,7 +468,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { return FillBlock(ancestor, ancestor_out, lock, active); } @@ -479,8 +479,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block = LookupBlockIndex(block_hash); - const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = g_chainman.m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; return FillBlock(ancestor, ancestor_out, lock, active); } @@ -488,8 +488,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block1 = LookupBlockIndex(block_hash1); - const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* block1 = g_chainman.m_blockman.LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = g_chainman.m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. @@ -499,7 +499,7 @@ public: double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); - return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); + return GuessVerificationProgress(Params().TxData(), g_chainman.m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override { @@ -511,7 +511,7 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low diff --git a/src/psbt.cpp b/src/psbt.cpp index 4db57d3cd0..a849b2ea53 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -363,7 +363,7 @@ bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base6 bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) { - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss_data(MakeUCharSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/qt/README.md b/src/qt/README.md index 30c68db15b..20c712c98d 100644 --- a/src/qt/README.md +++ b/src/qt/README.md @@ -1,10 +1,12 @@ -This directory contains the BitcoinQT graphical user interface (GUI). It uses the cross-platform framework [Qt](https://www1.qt.io/developers/). +This directory contains the source code for the Bitcoin Core graphical user interface (GUI). It uses the [Qt](https://www1.qt.io/developers/) cross-platform framework. The current precise version for Qt 5 is specified in [qt.mk](/depends/packages/qt.mk). ## Compile and run -See build instructions ([macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [Unix](/doc/build-unix.md), etc). +See build instructions: [Unix](/doc/build-unix.md), [macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [FreeBSD](/doc/build-freebsd.md), [NetBSD](/doc/build-netbsd.md), [OpenBSD](/doc/build-openbsd.md) + +When following your systems build instructions, make sure to install the `Qt` dependencies. To run: @@ -12,84 +14,111 @@ To run: ./src/qt/bitcoin-qt ``` -## Files and directories - -### forms +## Files and Directories -Contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. They are created with [Qt Creator](#using-qt-creator-as-ide), but can be edited using any text editor. +#### forms/ -### locale +- A directory that contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. These files specify the characteristics of form elements in XML. Qt UI files can be edited with [Qt Creator](#using-qt-creator-as-ide) or using any text editor. -Contains translations. They are periodically updated. The process is described [here](/doc/translation_process.md). +#### locale/ -### res +- Contains translations. They are periodically updated and an effort is made to support as many languages as possible. The process of contributing translations is described in [doc/translation_process.md](/doc/translation_process.md). -Resources such as the icon. +#### res/ -### test + - Contains graphical resources used to enhance the UI experience. -Tests. +#### test/ -### bitcoingui.(h/cpp) +- Functional tests used to ensure proper functionality of the GUI. Significant changes to the GUI code normally require new or updated tests. -Represents the main window of the Bitcoin UI. +#### bitcoingui.(h/cpp) -### \*model.(h/cpp) +- Represents the main window of the Bitcoin UI. -The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +#### \*model.(h/cpp) -ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. +- The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +- ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. -### \*page.(h/cpp) +#### \*page.(h/cpp) -A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. +- A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. -### \*dialog.(h/cpp) +#### \*dialog.(h/cpp) -Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). +- Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). -### paymentserver.(h/cpp) +#### paymentserver.(h/cpp) -Used to process BIP21 payment URI requests. Also handles URI based application switching (e.g. when following a bitcoin:... link from a browser). +- (Deprecated) Used to process BIP21 payment URI requests. Also handles URI-based application switching (e.g. when following a bitcoin:... link from a browser). -### walletview.(h/cpp) +#### walletview.(h/cpp) -Represents the view to a single wallet. +- Represents the view to a single wallet. -### Other .h/cpp files +#### Other .h/cpp files * UI elements like BitcoinAmountField, which inherit from QWidget. * `bitcoinstrings.cpp`: automatically generated -* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc handling +* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc. handling * `callback.h` -* `guiconstants.h`: UI colors, app name, etc +* `guiconstants.h`: UI colors, app name, etc. * `guiutil.h`: several helper functions * `macdockiconhandler.(h/mm)`: macOS dock icon handler * `macnotificationhandler.(h/mm)`: display notifications in macOS ## Contribute -See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. Specifically for Qt: +See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. + +**Note:** Do not change `local/bitcoin_en.ts`. It is updated [automatically](/doc/translation_process.md#writing-code-with-translations). + +## Using Qt Creator as an IDE + +[Qt Creator](https://www.qt.io/product/development-tools) is a powerful tool which packages a UI designer tool (Qt Designer) and a C++ IDE into one application. This is especially useful if you want to change the UI layout. + +#### Download Qt Creator + +On Unix and macOS, Qt Creator can be installed through your package manager. Alternatively, you can download a binary from the [Qt Website](https://www.qt.io/download/). + +**Note:** If installing from a binary grabbed from the Qt Website: During the installation process, uncheck everything except for `Qt Creator`. + +##### macOS + +```sh +brew install qt-creator +``` + +##### Ubuntu & Debian + +```sh +sudo apt-get install qtcreator +``` + +#### Setup Qt Creator -* don't change `local/bitcoin_en.ts`; this happens [automatically](/doc/translation_process.md#writing-code-with-translations) +1. Make sure you've installed all dependencies specified in your systems build instructions +2. Follow the compile instructions for your system, run `./configure` with the `--enable-debug` flag +3. Start Qt Creator. At the start page, do: `New` -> `Import Project` -> `Import Existing Project` +4. Enter `bitcoin-qt` as the Project Name and enter the absolute path to `src/qt` as Location +5. Check over the file selection, you may need to select the `forms` directory (necessary if you intend to edit *.ui files) +6. Confirm the `Summary` page +7. In the `Projects` tab, select `Manage Kits...` -## Using Qt Creator as IDE + **macOS** + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"Clang (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"LLDB"` as debugger (you might need to set the path to your LLDB installation) -You can use Qt Creator as an IDE. This is especially useful if you want to change -the UI layout. + **Ubuntu & Debian** -Download and install the community edition of [Qt Creator](https://www.qt.io/download/). -Uncheck everything except Qt Creator during the installation process. + Note: Some of these options may already be set -Instructions for macOS: + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"GCC (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"GDB"` as debugger -1. Make sure you installed everything through Homebrew mentioned in the [macOS build instructions](/doc/build-osx.md) -2. Use `./configure` with the `--enable-debug` flag -3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project -4. Enter "bitcoin-qt" as project name, enter src/qt as location -5. Leave the file selection as it is -6. Confirm the "summary page" -7. In the "Projects" tab select "Manage Kits..." -8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler -9. Select LLDB as debugger (you might need to set the path to your installation) -10. Start debugging with Qt Creator (you might need to the executable to "bitcoin-qt" under "Run", which is where you can also add command line arguments) +8. While in the `Projects` tab, ensure that you have the `bitcoin-qt` executable specified under `Run` + - If the executable is not specified: click `"Choose..."`, navigate to `src/qt`, and select `bitcoin-qt` +9. You're all set! Start developing, building, and debugging the Bitcoin Core GUI diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 0fa52359a8..ab6168a541 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -56,7 +56,7 @@ protected: }; AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, Tabs _tab, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AddressBookPage), model(nullptr), mode(_mode), diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index c2349b78f9..5b1330b81b 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -20,7 +20,7 @@ #include <QPushButton> AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureString* passphrase_out) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AskPassphraseDialog), mode(_mode), model(nullptr), diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8efb0e35d0..d6d5ba6968 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -51,6 +51,7 @@ #include <QThread> #include <QTimer> #include <QTranslator> +#include <QtGlobal> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -466,6 +467,13 @@ int GuiMain(int argc, char* argv[]) QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling); #endif +#if (QT_VERSION <= QT_VERSION_CHECK(5, 9, 8)) && defined(Q_OS_MACOS) + const auto os_name = QSysInfo::prettyProductName(); + if (os_name.startsWith("macOS 11") || os_name.startsWith("macOS 10.16")) { + QApplication::setStyle("fusion"); + } +#endif + BitcoinApplication app; /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index f2a49e5a76..ccc0a6828c 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,6 +30,7 @@ #include <qt/macdockiconhandler.h> #endif +#include <functional> #include <chain.h> #include <chainparams.h> #include <interfaces/handler.h> @@ -845,7 +846,7 @@ void BitcoinGUI::showDebugWindowActivateConsole() void BitcoinGUI::showHelpMessageClicked() { - helpMessageDialog->show(); + GUIUtil::bringToFront(helpMessageDialog); } #ifdef ENABLE_WALLET diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a2f46c339b..56813b2d19 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include <validation.h> #include <stdint.h> +#include <functional> #include <QDebug> #include <QThread> diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 08abef7866..ca78c96d70 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -42,7 +42,7 @@ bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { } CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _model, const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CoinControlDialog), m_coin_control(coin_control), model(_model), diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 70f080f4de..1467801522 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -9,10 +9,12 @@ #include <qt/createwalletdialog.h> #include <qt/forms/ui_createwalletdialog.h> +#include <qt/guiutil.h> + #include <QPushButton> CreateWalletDialog::CreateWalletDialog(QWidget* parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CreateWalletDialog) { ui->setupUi(this); diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index cd25f856a2..e51ed9e656 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -13,7 +13,7 @@ EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::EditAddressDialog), mapper(nullptr), mode(_mode), diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index 0b33c2cb8d..881869a46c 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -7,140 +7,135 @@ <x>0</x> <y>0</y> <width>364</width> - <height>213</height> + <height>249</height> </rect> </property> <property name="windowTitle"> <string>Create Wallet</string> </property> - <widget class="QDialogButtonBox" name="buttonBox"> - <property name="geometry"> - <rect> - <x>10</x> - <y>170</y> - <width>341</width> - <height>32</height> - </rect> - </property> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - <widget class="QLineEdit" name="wallet_name_line_edit"> - <property name="geometry"> - <rect> - <x>120</x> - <y>20</y> - <width>231</width> - <height>24</height> - </rect> - </property> - <property name="placeholderText"> - <string>Wallet</string> - </property> - </widget> - <widget class="QLabel" name="label"> - <property name="geometry"> - <rect> - <x>20</x> - <y>20</y> - <width>101</width> - <height>21</height> - </rect> - </property> - <property name="text"> - <string>Wallet Name</string> - </property> - </widget> - <widget class="QCheckBox" name="encrypt_wallet_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>50</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Encrypt the wallet. The wallet will be encrypted with a passphrase of your choice.</string> - </property> - <property name="text"> - <string>Encrypt Wallet</string> - </property> - <property name="checked"> - <bool>false</bool> - </property> - </widget> - <widget class="QLabel" name="advanced_options_label"> - <property name="geometry"> - <rect> - <x>20</x> - <y>90</y> - <width>220</width> - <height>21</height> - </rect> - </property> - <property name="styleSheet"> - <string notr="true">font-weight:bold;</string> - </property> - <property name="text"> - <string>Advanced options</string> - </property> - </widget> - <widget class="QCheckBox" name="disable_privkeys_checkbox"> - <property name="enabled"> - <bool>true</bool> - </property> - <property name="geometry"> - <rect> - <x>20</x> - <y>115</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Disable private keys for this wallet. Wallets with private keys disabled will have no private keys and cannot have an HD seed or imported private keys. This is ideal for watch-only wallets.</string> - </property> - <property name="text"> - <string>Disable Private Keys</string> - </property> - </widget> - <widget class="QCheckBox" name="blank_wallet_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>135</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Make a blank wallet. Blank wallets do not initially have private keys or scripts. Private keys and addresses can be imported, or an HD seed can be set, at a later time.</string> - </property> - <property name="text"> - <string>Make Blank Wallet</string> - </property> - </widget> - <widget class="QCheckBox" name="descriptor_checkbox"> - <property name="geometry"> - <rect> - <x>20</x> - <y>155</y> - <width>220</width> - <height>22</height> - </rect> - </property> - <property name="toolTip"> - <string>Use descriptors for scriptPubKey management</string> - </property> - <property name="text"> - <string>Descriptor Wallet</string> - </property> - </widget> + <property name="sizeGripEnabled"> + <bool>true</bool> + </property> + <layout class="QVBoxLayout" name="verticalLayout"> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <widget class="QLabel" name="wallet_name_label"> + <property name="text"> + <string>Wallet Name</string> + </property> + </widget> + </item> + <item> + <widget class="QLineEdit" name="wallet_name_line_edit"> + <property name="minimumSize"> + <size> + <width>262</width> + <height>0</height> + </size> + </property> + <property name="placeholderText"> + <string>Wallet</string> + </property> + </widget> + </item> + </layout> + </item> + <item> + <widget class="QCheckBox" name="encrypt_wallet_checkbox"> + <property name="toolTip"> + <string>Encrypt the wallet. The wallet will be encrypted with a passphrase of your choice.</string> + </property> + <property name="text"> + <string>Encrypt Wallet</string> + </property> + <property name="checked"> + <bool>false</bool> + </property> + </widget> + </item> + <item> + <spacer name="verticalSpacer_1"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeType"> + <enum>QSizePolicy::Fixed</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>8</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QGroupBox" name="groupBox"> + <property name="title"> + <string>Advanced Options</string> + </property> + <layout class="QVBoxLayout" name="verticalLayout_groupbox"> + <item> + <widget class="QCheckBox" name="disable_privkeys_checkbox"> + <property name="enabled"> + <bool>true</bool> + </property> + <property name="toolTip"> + <string>Disable private keys for this wallet. Wallets with private keys disabled will have no private keys and cannot have an HD seed or imported private keys. This is ideal for watch-only wallets.</string> + </property> + <property name="text"> + <string>Disable Private Keys</string> + </property> + </widget> + </item> + <item> + <widget class="QCheckBox" name="blank_wallet_checkbox"> + <property name="toolTip"> + <string>Make a blank wallet. Blank wallets do not initially have private keys or scripts. Private keys and addresses can be imported, or an HD seed can be set, at a later time.</string> + </property> + <property name="text"> + <string>Make Blank Wallet</string> + </property> + </widget> + </item> + <item> + <widget class="QCheckBox" name="descriptor_checkbox"> + <property name="toolTip"> + <string>Use descriptors for scriptPubKey management</string> + </property> + <property name="text"> + <string>Descriptor Wallet</string> + </property> + </widget> + </item> + </layout> + </widget> + </item> + <item> + <spacer name="verticalSpacer_2"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>0</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QDialogButtonBox" name="buttonBox"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="standardButtons"> + <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> + </property> + </widget> + </item> + </layout> </widget> <tabstops> <tabstop>wallet_name_line_edit</tabstop> diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index bce578a158..3831852185 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1079,7 +1079,7 @@ <item row="1" column="0"> <widget class="QLabel" name="peerConnectionTypeLabel"> <property name="toolTip"> - <string>The type of peer connection:<ul><li>Inbound: initiated by peer</li><li>Outbound Full Relay: default</li><li>Outbound Block Relay: does not relay transactions or addresses</li><li>Outbound Manual: added using RPC %1 or %2/%3 configuration options</li><li>Outbound Feeler: short-lived, for testing addresses</li><li>Outbound Address Fetch: short-lived, for soliciting addresses</li></ul></string> + <string>The type of peer connection: %1</string> </property> <property name="text"> <string>Connection Type</string> diff --git a/src/qt/forms/overviewpage.ui b/src/qt/forms/overviewpage.ui index 4d3f90c484..ee9d4a113c 100644 --- a/src/qt/forms/overviewpage.ui +++ b/src/qt/forms/overviewpage.ui @@ -504,7 +504,7 @@ </layout> </item> <item> - <widget class="QListView" name="listTransactions"> + <widget class="TransactionOverviewWidget" name="listTransactions"> <property name="styleSheet"> <string notr="true">QListView { background: transparent; }</string> </property> @@ -517,9 +517,15 @@ <property name="horizontalScrollBarPolicy"> <enum>Qt::ScrollBarAlwaysOff</enum> </property> + <property name="sizeAdjustPolicy"> + <enum>QAbstractScrollArea::AdjustToContents</enum> + </property> <property name="selectionMode"> <enum>QAbstractItemView::NoSelection</enum> </property> + <property name="uniformItemSizes"> + <bool>true</bool> + </property> </widget> </item> </layout> @@ -544,6 +550,13 @@ </item> </layout> </widget> + <customwidgets> + <customwidget> + <class>TransactionOverviewWidget</class> + <extends>QListView</extends> + <header>qt/transactionoverviewwidget.h</header> + </customwidget> + </customwidgets> <resources/> <connections/> </ui> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 430ecd322f..0c2bbe9516 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -40,12 +40,14 @@ #include <QFontDatabase> #include <QFontMetrics> #include <QGuiApplication> +#include <QJsonObject> #include <QKeyEvent> #include <QLineEdit> #include <QList> #include <QLocale> #include <QMenu> #include <QMouseEvent> +#include <QPluginLoader> #include <QProgressDialog> #include <QScreen> #include <QSettings> @@ -936,6 +938,20 @@ void LogQtInfo() const std::string plugin_link{"dynamic"}; #endif LogPrintf("Qt %s (%s), plugin=%s (%s)\n", qVersion(), qt_link, QGuiApplication::platformName().toStdString(), plugin_link); + const auto static_plugins = QPluginLoader::staticPlugins(); + if (static_plugins.empty()) { + LogPrintf("No static plugins.\n"); + } else { + LogPrintf("Static plugins:\n"); + for (const QStaticPlugin& p : static_plugins) { + QJsonObject meta_data = p.metaData(); + const std::string plugin_class = meta_data.take(QString("className")).toString().toStdString(); + const int plugin_version = meta_data.take(QString("version")).toInt(); + LogPrintf(" %s, version %d\n", plugin_class, plugin_version); + } + } + + LogPrintf("Style: %s / %s\n", QApplication::style()->objectName().toStdString(), QApplication::style()->metaObject()->className()); LogPrintf("System: %s, %s\n", QSysInfo::prettyProductName().toStdString(), QSysInfo::buildAbi().toStdString()); for (const QScreen* s : QGuiApplication::screens()) { LogPrintf("Screen: %s %dx%d, pixel ratio=%.1f\n", s->name().toStdString(), s->size().width(), s->size().height(), s->devicePixelRatio()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index c471b888f7..9fb108dff9 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -45,6 +45,9 @@ QT_END_NAMESPACE */ namespace GUIUtil { + // Use this flags to prevent a "What's This" button in the title bar of the dialog on Windows. + constexpr auto dialog_flags = Qt::WindowTitleHint | Qt::WindowSystemMenuHint | Qt::WindowCloseButtonHint; + // Create human-readable string from date QString dateTimeStr(const QDateTime &datetime); QString dateTimeStr(qint64 nTime); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 235722d091..aa6b2665fa 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -119,7 +119,7 @@ int GetPruneTargetGB() } // namespace Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::Intro), thread(nullptr), signalled(false), diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 9a3d43c2a6..10bf82d532 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -11,7 +11,7 @@ #include <QUrl> OpenURIDialog::OpenURIDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OpenURIDialog) { ui->setupUi(this); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 416b9c83c9..7b8d7871ec 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -29,7 +29,7 @@ #include <QTimer> OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OptionsDialog), model(nullptr), mapper(nullptr) diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index cca4dce624..bc542a0833 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -12,6 +12,7 @@ #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/transactionfilterproxy.h> +#include <qt/transactionoverviewwidget.h> #include <qt/transactiontablemodel.h> #include <qt/walletmodel.h> @@ -21,6 +22,9 @@ #include <QPainter> #include <QStatusTipEvent> +#include <algorithm> +#include <map> + #define DECORATION_SIZE 54 #define NUM_ITEMS 5 @@ -34,7 +38,7 @@ public: QAbstractItemDelegate(parent), unit(BitcoinUnits::BTC), platformStyle(_platformStyle) { - + connect(this, &TxViewDelegate::width_changed, this, &TxViewDelegate::sizeHintChanged); } inline void paint(QPainter *painter, const QStyleOptionViewItem &option, @@ -67,13 +71,15 @@ public: painter->setPen(foreground); QRect boundingRect; - painter->drawText(addressRect, Qt::AlignLeft|Qt::AlignVCenter, address, &boundingRect); + painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); + int address_rect_min_width = boundingRect.width(); if (index.data(TransactionTableModel::WatchonlyRole).toBool()) { QIcon iconWatchonly = qvariant_cast<QIcon>(index.data(TransactionTableModel::WatchonlyDecorationRole)); QRect watchonlyRect(boundingRect.right() + 5, mainRect.top()+ypad+halfheight, 16, halfheight); iconWatchonly.paint(painter, watchonlyRect); + address_rect_min_width += 5 + watchonlyRect.width(); } if(amount < 0) @@ -94,23 +100,42 @@ public: { amountText = QString("[") + amountText + QString("]"); } - painter->drawText(amountRect, Qt::AlignRight|Qt::AlignVCenter, amountText); + + QRect amount_bounding_rect; + painter->drawText(amountRect, Qt::AlignRight | Qt::AlignVCenter, amountText, &amount_bounding_rect); painter->setPen(option.palette.color(QPalette::Text)); - painter->drawText(amountRect, Qt::AlignLeft|Qt::AlignVCenter, GUIUtil::dateTimeStr(date)); + QRect date_bounding_rect; + painter->drawText(amountRect, Qt::AlignLeft | Qt::AlignVCenter, GUIUtil::dateTimeStr(date), &date_bounding_rect); + + const int minimum_width = std::max(address_rect_min_width, amount_bounding_rect.width() + date_bounding_rect.width()); + const auto search = m_minimum_width.find(index.row()); + if (search == m_minimum_width.end() || search->second != minimum_width) { + m_minimum_width[index.row()] = minimum_width; + Q_EMIT width_changed(index); + } painter->restore(); } inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override { - return QSize(DECORATION_SIZE, DECORATION_SIZE); + const auto search = m_minimum_width.find(index.row()); + const int minimum_text_width = search == m_minimum_width.end() ? 0 : search->second; + return {DECORATION_SIZE + 8 + minimum_text_width, DECORATION_SIZE}; } int unit; - const PlatformStyle *platformStyle; +Q_SIGNALS: + //! An intermediate signal for emitting from the `paint() const` member function. + void width_changed(const QModelIndex& index) const; + +private: + const PlatformStyle* platformStyle; + mutable std::map<int, int> m_minimum_width; }; + #include <qt/overviewpage.moc> OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) : @@ -126,7 +151,6 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) // use a SingleColorIcon for the "out of sync warning" icon QIcon icon = platformStyle->SingleColorIcon(":/icons/warning"); - icon.addPixmap(icon.pixmap(QSize(64,64), QIcon::Normal), QIcon::Disabled); // also set the disabled icon because we are using a disabled QPushButton to work around missing HiDPI support of QLabel (https://bugreports.qt.io/browse/QTBUG-42503) ui->labelTransactionsStatus->setIcon(icon); ui->labelWalletStatus->setIcon(icon); @@ -136,7 +160,7 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) ui->listTransactions->setMinimumHeight(NUM_ITEMS * (DECORATION_SIZE + 2)); ui->listTransactions->setAttribute(Qt::WA_MacShowFocusRect, false); - connect(ui->listTransactions, &QListView::clicked, this, &OverviewPage::handleTransactionClicked); + connect(ui->listTransactions, &TransactionOverviewWidget::clicked, this, &OverviewPage::handleTransactionClicked); // start with displaying the "out of sync" warnings showOutOfSyncWarning(true); diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 58167d4bb4..55ab6046cf 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -19,7 +19,7 @@ PSBTOperationsDialog::PSBTOperationsDialog( - QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent), + QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), m_ui(new Ui::PSBTOperationsDialog), m_wallet_model(wallet_model), m_client_model(client_model) diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 1a9cd78d1e..0aea920c86 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -8,6 +8,7 @@ #include <qt/forms/ui_receivecoinsdialog.h> #include <qt/addresstablemodel.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/receiverequestdialog.h> @@ -21,7 +22,7 @@ #include <QTextDocument> ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveCoinsDialog), columnResizingFixer(nullptr), model(nullptr), diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index bef3edc5fe..78ae5c07da 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -19,7 +19,7 @@ #endif ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveRequestDialog), model(nullptr) { diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 18b913774b..03531a1381 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -181,7 +181,7 @@ void RecentRequestsTableModel::addNewRequest(const SendCoinsRecipient &recipient // called from ctor when loading from wallet void RecentRequestsTableModel::addNewRequest(const std::string &recipient) { - std::vector<char> data(recipient.begin(), recipient.end()); + std::vector<uint8_t> data(recipient.begin(), recipient.end()); CDataStream ss(data, SER_DISK, CLIENT_VERSION); RecentRequestEntry entry; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index df98dbbc99..2758ee351a 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -16,9 +16,10 @@ #include <chainparams.h> #include <interfaces/node.h> #include <netbase.h> -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> #include <util/threadnames.h> @@ -459,11 +460,22 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->splitter->restoreState(settings.value("PeersTabSplitterSizes").toByteArray()); - QChar nonbreaking_hyphen(8209); + constexpr QChar nonbreaking_hyphen(8209); + const std::vector<QString> CONNECTION_TYPE_DOC{ + tr("Inbound: initiated by peer"), + tr("Outbound Full Relay: default"), + tr("Outbound Block Relay: does not relay transactions or addresses"), + tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") + .arg("addnode") + .arg(QString(nonbreaking_hyphen) + "addnode") + .arg(QString(nonbreaking_hyphen) + "connect"), + tr("Outbound Feeler: short-lived, for testing addresses"), + tr("Outbound Address Fetch: short-lived, for soliciting addresses")}; + const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"}; + ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list)); ui->dataDir->setToolTip(ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg(QString(nonbreaking_hyphen) + "blocksdir")); ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); - ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg("addnode").arg(QString(nonbreaking_hyphen) + "addnode").arg(QString(nonbreaking_hyphen) + "connect")); if (platformStyle->getImagesOnButtons()) { ui->openDebugLogfileButton->setIcon(platformStyle->SingleColorIcon(":/icons/export")); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 9eedd8cf75..611f3584c3 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -55,7 +55,7 @@ int getIndexForConfTarget(int target) { } SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SendCoinsDialog), clientModel(nullptr), model(nullptr), diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 6c110f0688..2e7df60574 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -19,7 +19,7 @@ #include <QClipboard> SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SignVerifyMessageDialog), model(nullptr), platformStyle(_platformStyle) diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 5796a6ef56..2292c01d6a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -18,6 +18,8 @@ #include <util/system.h> #include <util/translation.h> +#include <functional> + #include <QApplication> #include <QCloseEvent> #include <QPainter> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 24d7735447..0d9928d363 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -33,7 +33,7 @@ static RPCHelpMan rpcNestedTest_rpc() } static const CRPCCommand vRPCCommands[] = { - {"test", "rpcNestedTest", &rpcNestedTest_rpc, {"arg1", "arg2", "arg3"}}, + {"test", &rpcNestedTest_rpc}, }; void RPCNestedTests::rpcNestedTests() diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index b547bf7645..6f31d8463f 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -11,7 +11,7 @@ #include <QModelIndex> TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::TransactionDescDialog) { ui->setupUi(this); diff --git a/src/qt/transactionoverviewwidget.h b/src/qt/transactionoverviewwidget.h new file mode 100644 index 0000000000..2bdead7bc4 --- /dev/null +++ b/src/qt/transactionoverviewwidget.h @@ -0,0 +1,41 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H +#define BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H + +#include <qt/transactiontablemodel.h> + +#include <QListView> +#include <QSize> +#include <QSizePolicy> + +QT_BEGIN_NAMESPACE +class QShowEvent; +class QWidget; +QT_END_NAMESPACE + +class TransactionOverviewWidget : public QListView +{ + Q_OBJECT + +public: + explicit TransactionOverviewWidget(QWidget* parent = nullptr) : QListView(parent) {} + + QSize sizeHint() const override + { + return {sizeHintForColumn(TransactionTableModel::ToAddress), QListView::sizeHint().height()}; + } + +protected: + void showEvent(QShowEvent* event) override + { + Q_UNUSED(event); + QSizePolicy sp = sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Minimum); + setSizePolicy(sp); + } +}; + +#endif // BITCOIN_QT_TRANSACTIONOVERVIEWWIDGET_H diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 05b250eea9..e6d483364c 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -19,6 +19,7 @@ #include <uint256.h> #include <algorithm> +#include <functional> #include <QColor> #include <QDateTime> diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index b7f85446f4..05499b2a41 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -29,7 +29,7 @@ /** "Help message" or "About" dialog box */ HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::HelpMessageDialog) { ui->setupUi(this); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 2edb7eff8a..02b3c62867 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -106,9 +106,24 @@ void WalletFrame::setCurrentWallet(WalletModel* wallet_model) { if (mapWalletViews.count(wallet_model) == 0) return; + // Stop the effect of hidden widgets on the size hint of the shown one in QStackedWidget. + WalletView* view_about_to_hide = currentWalletView(); + if (view_about_to_hide) { + QSizePolicy sp = view_about_to_hide->sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Ignored); + view_about_to_hide->setSizePolicy(sp); + } + WalletView *walletView = mapWalletViews.value(wallet_model); - walletStack->setCurrentWidget(walletView); assert(walletView); + + // Set or restore the default QSizePolicy which could be set to QSizePolicy::Ignored previously. + QSizePolicy sp = walletView->sizePolicy(); + sp.setHorizontalPolicy(QSizePolicy::Preferred); + walletView->setSizePolicy(sp); + walletView->updateGeometry(); + + walletStack->setCurrentWidget(walletView); walletView->updateEncryptionStatus(); } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index cad472b43b..02254da3ce 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -29,6 +29,7 @@ #include <wallet/wallet.h> // for CRecipient #include <stdint.h> +#include <functional> #include <QDebug> #include <QMessageBox> @@ -244,7 +245,7 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; - transaction_array.append(&(ssTx[0]), ssTx.size()); + transaction_array.append((const char*)&(ssTx[0]), ssTx.size()); } // Add addresses / update labels that we've sent to the address book, @@ -514,6 +515,13 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append("</td><td>"); questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee)); questionString.append("</td></tr></table>"); + + // Display warning in the "Confirm fee bump" window if the "Coin Control Features" option is enabled + if (getOptionsModel()->getCoinControlFeatures()) { + questionString.append("<br><br>"); + questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); + } + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); diff --git a/src/rest.cpp b/src/rest.cpp index 678ffdb760..f872f6e59d 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -179,7 +179,7 @@ static bool rest_headers(const util::Ref& context, { LOCK(cs_main); tip = ::ChainActive().Tip(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && ::ChainActive().Contains(pindex)) { headers.push_back(pindex); if (headers.size() == (unsigned long)count) @@ -247,7 +247,7 @@ static bool rest_block(HTTPRequest* req, { LOCK(cs_main); tip = ::ChainActive().Tip(); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 3f97f6f3d9..6e2a7c330e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -17,8 +17,8 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> -#include <policy/fees.h> #include <policy/feerate.h> +#include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> #include <primitives/transaction.h> @@ -156,21 +156,11 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) { - // Serialize passed information without accessing chain state of the active chain! - AssertLockNotHeld(cs_main); // For performance reasons + UniValue result = blockheaderToJSON(tip, blockindex); - UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); - const CBlockIndex* pnext; - int confirmations = ComputeNextBlockAndDepth(tip, blockindex, pnext); - result.pushKV("confirmations", confirmations); result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", block.nVersion); - result.pushKV("versionHex", strprintf("%08x", block.nVersion)); - result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); UniValue txs(UniValue::VARR); if (txDetails) { CBlockUndo blockUndo; @@ -189,18 +179,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn } } result.pushKV("tx", txs); - result.pushKV("time", block.GetBlockTime()); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)block.nNonce); - result.pushKV("bits", strprintf("%08x", block.nBits)); - result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); - result.pushKV("nTx", (uint64_t)blockindex->nTx); - if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); - if (pnext) - result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } @@ -843,7 +822,7 @@ static RPCHelpMan getblockheader() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); } @@ -967,7 +946,7 @@ static RPCHelpMan getblock() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); if (!pblockindex) { @@ -1164,7 +1143,7 @@ static RPCHelpMan gettxout() } } - const CBlockIndex* pindex = LookupBlockIndex(coins_view->GetBestBlock()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(coins_view->GetBestBlock()); ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); if (coin.nHeight == MEMPOOL_HEIGHT) { ret.pushKV("confirmations", 0); @@ -1500,6 +1479,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); + ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); @@ -1520,6 +1500,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::NUM, "size", "Current tx count"}, {RPCResult::Type::NUM, "bytes", "Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted"}, {RPCResult::Type::NUM, "usage", "Total memory usage for the mempool"}, + {RPCResult::Type::STR_AMOUNT, "total_fee", "Total fees for the mempool in " + CURRENCY_UNIT + ", ignoring modified fees through prioritizetransaction"}, {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, @@ -1557,7 +1538,7 @@ static RPCHelpMan preciousblock() { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1595,7 +1576,7 @@ static RPCHelpMan invalidateblock() CBlockIndex* pblockindex; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1603,7 +1584,7 @@ static RPCHelpMan invalidateblock() InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); } if (!state.IsValid()) { @@ -1634,7 +1615,7 @@ static RPCHelpMan reconsiderblock() { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + CBlockIndex* pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1643,7 +1624,7 @@ static RPCHelpMan reconsiderblock() } BlockValidationState state; - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); @@ -1689,7 +1670,7 @@ static RPCHelpMan getchaintxstats() } else { uint256 hash(ParseHashV(request.params[1], "blockhash")); LOCK(cs_main); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1867,7 +1848,7 @@ static RPCHelpMan getblockstats() pindex = ::ChainActive()[height]; } else { const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2330,7 +2311,7 @@ static RPCHelpMan getblockfilter() bool block_was_connected; { LOCK(cs_main); - block_index = LookupBlockIndex(block_hash); + block_index = g_chainman.m_blockman.LookupBlockIndex(block_hash); if (!block_index) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2399,10 +2380,10 @@ static RPCHelpMan dumptxoutset() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + const fs::path path = fsbridge::AbsPathJoin(GetDataDir(), request.params[0].get_str()); // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + const fs::path temppath = fsbridge::AbsPathJoin(GetDataDir(), request.params[0].get_str() + ".incomplete"); if (fs::exists(path)) { throw JSONRPCError( @@ -2440,7 +2421,7 @@ static RPCHelpMan dumptxoutset() } pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - tip = LookupBlockIndex(stats.hashBlock); + tip = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock); CHECK_NONFATAL(tip); } @@ -2480,41 +2461,41 @@ void RegisterBlockchainRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "blockchain", "getblockchaininfo", &getblockchaininfo, {} }, - { "blockchain", "getchaintxstats", &getchaintxstats, {"nblocks", "blockhash"} }, - { "blockchain", "getblockstats", &getblockstats, {"hash_or_height", "stats"} }, - { "blockchain", "getbestblockhash", &getbestblockhash, {} }, - { "blockchain", "getblockcount", &getblockcount, {} }, - { "blockchain", "getblock", &getblock, {"blockhash","verbosity|verbose"} }, - { "blockchain", "getblockhash", &getblockhash, {"height"} }, - { "blockchain", "getblockheader", &getblockheader, {"blockhash","verbose"} }, - { "blockchain", "getchaintips", &getchaintips, {} }, - { "blockchain", "getdifficulty", &getdifficulty, {} }, - { "blockchain", "getmempoolancestors", &getmempoolancestors, {"txid","verbose"} }, - { "blockchain", "getmempooldescendants", &getmempooldescendants, {"txid","verbose"} }, - { "blockchain", "getmempoolentry", &getmempoolentry, {"txid"} }, - { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, - { "blockchain", "getrawmempool", &getrawmempool, {"verbose", "mempool_sequence"} }, - { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, - { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, - { "blockchain", "savemempool", &savemempool, {} }, - { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, - - { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, - { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, - { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "blockchain", &getblockchaininfo, }, + { "blockchain", &getchaintxstats, }, + { "blockchain", &getblockstats, }, + { "blockchain", &getbestblockhash, }, + { "blockchain", &getblockcount, }, + { "blockchain", &getblock, }, + { "blockchain", &getblockhash, }, + { "blockchain", &getblockheader, }, + { "blockchain", &getchaintips, }, + { "blockchain", &getdifficulty, }, + { "blockchain", &getmempoolancestors, }, + { "blockchain", &getmempooldescendants, }, + { "blockchain", &getmempoolentry, }, + { "blockchain", &getmempoolinfo, }, + { "blockchain", &getrawmempool, }, + { "blockchain", &gettxout, }, + { "blockchain", &gettxoutsetinfo, }, + { "blockchain", &pruneblockchain, }, + { "blockchain", &savemempool, }, + { "blockchain", &verifychain, }, + + { "blockchain", &preciousblock, }, + { "blockchain", &scantxoutset, }, + { "blockchain", &getblockfilter, }, /* Not shown in help */ - { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, - { "hidden", "reconsiderblock", &reconsiderblock, {"blockhash"} }, - { "hidden", "waitfornewblock", &waitfornewblock, {"timeout"} }, - { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, - { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, - { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, - { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, + { "hidden", &invalidateblock, }, + { "hidden", &reconsiderblock, }, + { "hidden", &waitfornewblock, }, + { "hidden", &waitforblock, }, + { "hidden", &waitforblockheight, }, + { "hidden", &syncwithvalidationinterfacequeue, }, + { "hidden", &dumptxoutset, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 042005b9a6..d1eb849b7e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -210,14 +210,9 @@ public: CRPCConvertTable::CRPCConvertTable() { - const unsigned int n_elem = - (sizeof(vRPCConvertParams) / sizeof(vRPCConvertParams[0])); - - for (unsigned int i = 0; i < n_elem; i++) { - members.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramIdx)); - membersByName.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramName)); + for (const auto& cp : vRPCConvertParams) { + members.emplace(cp.methodName, cp.paramIdx); + membersByName.emplace(cp.methodName, cp.paramName); } } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 0e96beb6db..50987a735b 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -375,7 +375,7 @@ static RPCHelpMan generateblock() LOCK(cs_main); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), block, g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) { throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); } } @@ -618,7 +618,7 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); uint256 hash = block.GetHash(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; @@ -632,7 +632,7 @@ static RPCHelpMan getblocktemplate() if (block.hashPrevBlock != pindexPrev->GetBlockHash()) return "inconclusive-not-best-prevblk"; BlockValidationState state; - TestBlockValidity(state, Params(), block, pindexPrev, false, true); + TestBlockValidity(state, Params(), ::ChainstateActive(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } @@ -966,7 +966,7 @@ static RPCHelpMan submitblock() uint256 hash = block.GetHash(); { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { return "duplicate"; @@ -979,7 +979,7 @@ static RPCHelpMan submitblock() { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(block.hashPrevBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); if (pindex) { UpdateUncommittedBlockStructures(block, pindex, Params().GetConsensus()); } @@ -1023,7 +1023,7 @@ static RPCHelpMan submitheader() } { LOCK(cs_main); - if (!LookupBlockIndex(h.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(h.hashPrevBlock)) { throw JSONRPCError(RPC_VERIFY_ERROR, "Must submit previous header (" + h.hashPrevBlock.GetHex() + ") first"); } } @@ -1232,24 +1232,24 @@ void RegisterMiningRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "mining", "getnetworkhashps", &getnetworkhashps, {"nblocks","height"} }, - { "mining", "getmininginfo", &getmininginfo, {} }, - { "mining", "prioritisetransaction", &prioritisetransaction, {"txid","dummy","fee_delta"} }, - { "mining", "getblocktemplate", &getblocktemplate, {"template_request"} }, - { "mining", "submitblock", &submitblock, {"hexdata","dummy"} }, - { "mining", "submitheader", &submitheader, {"hexdata"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "mining", &getnetworkhashps, }, + { "mining", &getmininginfo, }, + { "mining", &prioritisetransaction, }, + { "mining", &getblocktemplate, }, + { "mining", &submitblock, }, + { "mining", &submitheader, }, - { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, - { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, - { "generating", "generateblock", &generateblock, {"output","transactions"} }, + { "generating", &generatetoaddress, }, + { "generating", &generatetodescriptor, }, + { "generating", &generateblock, }, - { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, + { "util", &estimatesmartfee, }, - { "hidden", "estimaterawfee", &estimaterawfee, {"conf_target", "threshold"} }, - { "hidden", "generate", &generate, {} }, + { "hidden", &estimaterawfee, }, + { "hidden", &generate, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index b3102a236d..38a0bddddb 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -39,13 +39,14 @@ static RPCHelpMan validateaddress() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not. If not, this is the only property returned."}, + {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, {RPCResult::Type::STR, "address", "The bitcoin address validated"}, {RPCResult::Type::STR_HEX, "scriptPubKey", "The hex-encoded scriptPubKey generated by the address"}, {RPCResult::Type::BOOL, "isscript", "If the key is a script"}, {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address"}, {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, + {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, } }, RPCExamples{ @@ -54,13 +55,14 @@ static RPCHelpMan validateaddress() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - bool isValid = IsValidDestination(dest); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + const bool isValid = IsValidDestination(dest); + CHECK_NONFATAL(isValid == error_msg.empty()); UniValue ret(UniValue::VOBJ); ret.pushKV("isvalid", isValid); - if (isValid) - { + if (isValid) { std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -69,7 +71,10 @@ static RPCHelpMan validateaddress() UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); + } else { + ret.pushKV("error", error_msg); } + return ret; }, }; @@ -360,13 +365,13 @@ static RPCHelpMan signmessagewithprivkey() static RPCHelpMan setmocktime() { return RPCHelpMan{"setmocktime", - "\nSet the local time to given timestamp (-regtest only)\n", - { - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" - " Pass 0 to go back to using the system time."}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{""}, + "\nSet the local time to given timestamp (-regtest only)\n", + { + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" + "Pass 0 to go back to using the system time."}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { if (!Params().IsMockableChain()) { @@ -381,7 +386,10 @@ static RPCHelpMan setmocktime() LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - int64_t time = request.params[0].get_int64(); + const int64_t time{request.params[0].get_int64()}; + if (time < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime can not be negative: %s.", time)); + } SetMockTime(time); if (request.context.Has<NodeContext>()) { for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { @@ -694,23 +702,23 @@ void RegisterMiscRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "control", "getmemoryinfo", &getmemoryinfo, {"mode"} }, - { "control", "logging", &logging, {"include", "exclude"}}, - { "util", "validateaddress", &validateaddress, {"address"} }, - { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, - { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "range"} }, - { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, - { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, - { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, - { "util", "getindexinfo", &getindexinfo, {"index_name"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "control", &getmemoryinfo, }, + { "control", &logging, }, + { "util", &validateaddress, }, + { "util", &createmultisig, }, + { "util", &deriveaddresses, }, + { "util", &getdescriptorinfo, }, + { "util", &verifymessage, }, + { "util", &signmessagewithprivkey, }, + { "util", &getindexinfo, }, /* Not shown in help */ - { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, - { "hidden", "mockscheduler", &mockscheduler, {"delta_time"}}, - { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echojson, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", &setmocktime, }, + { "hidden", &mockscheduler, }, + { "hidden", &echo, }, + { "hidden", &echojson, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index cfca8b4ad4..47d77b341a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -941,24 +941,24 @@ void RegisterNetRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "network", "getconnectioncount", &getconnectioncount, {} }, - { "network", "ping", &ping, {} }, - { "network", "getpeerinfo", &getpeerinfo, {} }, - { "network", "addnode", &addnode, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, {"address", "nodeid"} }, - { "network", "getaddednodeinfo", &getaddednodeinfo, {"node"} }, - { "network", "getnettotals", &getnettotals, {} }, - { "network", "getnetworkinfo", &getnetworkinfo, {} }, - { "network", "setban", &setban, {"subnet", "command", "bantime", "absolute"} }, - { "network", "listbanned", &listbanned, {} }, - { "network", "clearbanned", &clearbanned, {} }, - { "network", "setnetworkactive", &setnetworkactive, {"state"} }, - { "network", "getnodeaddresses", &getnodeaddresses, {"count"} }, - - { "hidden", "addconnection", &addconnection, {"address", "connection_type"} }, - { "hidden", "addpeeraddress", &addpeeraddress, {"address", "port"} }, +{ // category actor + // --------------------- ----------------------- + { "network", &getconnectioncount, }, + { "network", &ping, }, + { "network", &getpeerinfo, }, + { "network", &addnode, }, + { "network", &disconnectnode, }, + { "network", &getaddednodeinfo, }, + { "network", &getnettotals, }, + { "network", &getnetworkinfo, }, + { "network", &setban, }, + { "network", &listbanned, }, + { "network", &clearbanned, }, + { "network", &setnetworkactive, }, + { "network", &getnodeaddresses, }, + + { "hidden", &addconnection, }, + { "hidden", &addpeeraddress, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index c8ceb2c186..fc00a1efad 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -79,6 +79,7 @@ enum RPCErrorCode RPC_WALLET_ALREADY_UNLOCKED = -17, //!< Wallet is already unlocked RPC_WALLET_NOT_FOUND = -18, //!< Invalid wallet specified RPC_WALLET_NOT_SPECIFIED = -19, //!< No wallet specified (error when there are multiple wallets loaded) + RPC_WALLET_ALREADY_LOADED = -35, //!< This same wallet is already loaded //! Backwards compatible aliases RPC_WALLET_INVALID_ACCOUNT_NAME = RPC_WALLET_INVALID_LABEL_NAME, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e2897549b5..ac42404470 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -54,7 +54,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& LOCK(cs_main); entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = LookupBlockIndex(hashBlock); + CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (::ChainActive().Contains(pindex)) { entry.pushKV("confirmations", 1 + ::ChainActive().Height() - pindex->nHeight); @@ -178,7 +178,7 @@ static RPCHelpMan getrawtransaction() LOCK(cs_main); uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); - blockindex = LookupBlockIndex(blockhash); + blockindex = g_chainman.m_blockman.LookupBlockIndex(blockhash); if (!blockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } @@ -260,7 +260,7 @@ static RPCHelpMan gettxoutproof() if (!request.params[1].isNull()) { LOCK(cs_main); hashBlock = ParseHashV(request.params[1], "blockhash"); - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -290,7 +290,7 @@ static RPCHelpMan gettxoutproof() if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); } @@ -350,7 +350,7 @@ static RPCHelpMan verifytxoutproof() LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(merkleBlock.header.GetHash()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); if (!pindex || !::ChainActive().Contains(pindex) || pindex->nTx == 0) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -816,10 +816,11 @@ static RPCHelpMan sendrawtransaction() { return RPCHelpMan{"sendrawtransaction", "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" - "\nNote that the transaction will be sent unconditionally to all peers, so using this\n" + "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" - "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", + "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" + "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK()), @@ -894,6 +895,7 @@ static RPCHelpMan testmempoolaccept() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, {RPCResult::Type::BOOL, "allowed", "If the mempool allows this tx to be inserted"}, {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, {RPCResult::Type::OBJ, "fees", "Transaction fees (only present if 'allowed' is true)", @@ -930,7 +932,6 @@ static RPCHelpMan testmempoolaccept() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed. Make sure the tx has at least one input."); } CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - const uint256& tx_hash = tx->GetHash(); const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? DEFAULT_MAX_RAW_TX_FEE_RATE : @@ -942,7 +943,8 @@ static RPCHelpMan testmempoolaccept() UniValue result(UniValue::VARR); UniValue result_0(UniValue::VOBJ); - result_0.pushKV("txid", tx_hash.GetHex()); + result_0.pushKV("txid", tx->GetHash().GetHex()); + result_0.pushKV("wtxid", tx->GetWitnessHash().GetHex()); TxValidationState state; bool test_accept_res; @@ -1343,7 +1345,7 @@ static RPCHelpMan combinepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1482,7 +1484,7 @@ static RPCHelpMan createpsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1551,7 +1553,7 @@ static RPCHelpMan converttopsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1642,7 +1644,7 @@ static RPCHelpMan utxoupdatepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1738,7 +1740,7 @@ static RPCHelpMan joinpsbts() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << shuffled_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1858,27 +1860,27 @@ void RegisterRawTransactionRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose","blockhash"} }, - { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, - { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, - { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, - { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, - { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, - { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, - { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, - { "rawtransactions", "createpsbt", &createpsbt, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "converttopsbt", &converttopsbt, {"hexstring","permitsigdata","iswitness"} }, - { "rawtransactions", "utxoupdatepsbt", &utxoupdatepsbt, {"psbt", "descriptors"} }, - { "rawtransactions", "joinpsbts", &joinpsbts, {"txs"} }, - { "rawtransactions", "analyzepsbt", &analyzepsbt, {"psbt"} }, - - { "blockchain", "gettxoutproof", &gettxoutproof, {"txids", "blockhash"} }, - { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "rawtransactions", &getrawtransaction, }, + { "rawtransactions", &createrawtransaction, }, + { "rawtransactions", &decoderawtransaction, }, + { "rawtransactions", &decodescript, }, + { "rawtransactions", &sendrawtransaction, }, + { "rawtransactions", &combinerawtransaction, }, + { "rawtransactions", &signrawtransactionwithkey, }, + { "rawtransactions", &testmempoolaccept, }, + { "rawtransactions", &decodepsbt, }, + { "rawtransactions", &combinepsbt, }, + { "rawtransactions", &finalizepsbt, }, + { "rawtransactions", &createpsbt, }, + { "rawtransactions", &converttopsbt, }, + { "rawtransactions", &utxoupdatepsbt, }, + { "rawtransactions", &joinpsbts, }, + { "rawtransactions", &analyzepsbt, }, + + { "blockchain", &gettxoutproof, }, + { "blockchain", &verifytxoutproof, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index f32d9abac6..9a9b3713f3 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -144,8 +144,13 @@ static RPCHelpMan help() [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue { std::string strCommand; - if (jsonRequest.params.size() > 0) + if (jsonRequest.params.size() > 0) { strCommand = jsonRequest.params[0].get_str(); + } + if (strCommand == "dump_all_command_conversions") { + // Used for testing only, undocumented + return tableRPC.dumpArgMap(); + } return tableRPC.help(strCommand, jsonRequest); }, @@ -244,13 +249,13 @@ static RPCHelpMan getrpcinfo() // clang-format off static const CRPCCommand vRPCCommands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- +{ // category actor (function) + // --------------------- ----------------------- /* Overall control/query calls */ - { "control", "getrpcinfo", &getrpcinfo, {} }, - { "control", "help", &help, {"command"} }, - { "control", "stop", &stop, {"wait"} }, - { "control", "uptime", &uptime, {} }, + { "control", &getrpcinfo, }, + { "control", &help, }, + { "control", &stop, }, + { "control", &uptime, }, }; // clang-format on @@ -479,6 +484,18 @@ std::vector<std::string> CRPCTable::listCommands() const return commandList; } +UniValue CRPCTable::dumpArgMap() const +{ + UniValue ret{UniValue::VARR}; + for (const auto& cmd : mapCommands) { + for (const auto& c : cmd.second) { + const auto help = RpcMethodFnType(c->unique_id)(); + help.AppendArgMap(ret); + } + } + return ret; +} + void RPCSetTimerInterfaceIfUnset(RPCTimerInterface *iface) { if (!timerInterface) diff --git a/src/rpc/server.h b/src/rpc/server.h index 040192b455..fe5a791e1e 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -103,7 +103,7 @@ public: } //! Simplified constructor taking plain RpcMethodFnType function pointer. - CRPCCommand(std::string category, std::string name_in, RpcMethodFnType fn, std::vector<std::string> args_in) + CRPCCommand(std::string category, RpcMethodFnType fn) : CRPCCommand( category, fn().m_name, @@ -111,8 +111,6 @@ public: fn().GetArgNames(), intptr_t(fn)) { - CHECK_NONFATAL(fn().m_name == name_in); - CHECK_NONFATAL(fn().GetArgNames() == args_in); } std::string category; @@ -147,6 +145,10 @@ public: */ std::vector<std::string> listCommands() const; + /** + * Return all named arguments that need to be converted by the client from string to another JSON type + */ + UniValue dumpArgMap() const; /** * Appends a CRPCCommand to the dispatch table. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 31072114da..bfdba5253c 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -549,6 +549,24 @@ std::string RPCHelpMan::ToString() const return ret; } +void RPCHelpMan::AppendArgMap(UniValue& arr) const +{ + for (int i{0}; i < int(m_args.size()); ++i) { + const auto& arg = m_args.at(i); + std::vector<std::string> arg_names; + boost::split(arg_names, arg.m_names, boost::is_any_of("|")); + for (const auto& arg_name : arg_names) { + UniValue map{UniValue::VARR}; + map.push_back(m_name); + map.push_back(i); + map.push_back(arg_name); + map.push_back(arg.m_type == RPCArg::Type::STR || + arg.m_type == RPCArg::Type::STR_HEX); + arr.push_back(map); + } + } +} + std::string RPCArg::GetFirstName() const { return m_names.substr(0, m_names.find("|")); diff --git a/src/rpc/util.h b/src/rpc/util.h index 942c243718..444a013ca1 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -336,6 +336,8 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); std::string ToString() const; + /** Append the named args that need to be converted from string to another JSON type */ + void AppendArgMap(UniValue& arr) const; UniValue HandleRequest(const JSONRPCRequest& request) { Check(request); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 7c361bf26f..b3ee23f139 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -7,6 +7,7 @@ #include <random.h> #include <assert.h> +#include <functional> #include <utility> CScheduler::CScheduler() diff --git a/src/scheduler.h b/src/scheduler.h index d7fe00d1b4..9eec8c0fa0 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -9,6 +9,7 @@ #include <functional> #include <list> #include <map> +#include <thread> #include <sync.h> @@ -35,6 +36,8 @@ public: CScheduler(); ~CScheduler(); + std::thread m_service_thread; + typedef std::function<void()> Function; /** Call func at/after time t */ @@ -62,8 +65,7 @@ public: void MockForward(std::chrono::seconds delta_seconds); /** - * Services the queue 'forever'. Should be run in a thread, - * and interrupted using boost::interrupt_thread + * Services the queue 'forever'. Should be run in a thread. */ void serviceQueue(); @@ -72,12 +74,14 @@ public: { WITH_LOCK(newTaskMutex, stopRequested = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** Tell any threads running serviceQueue to stop when there is no work left to be done */ void StopWhenDrained() { WITH_LOCK(newTaskMutex, stopWhenEmpty = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 2bfe206597..cf47d37e70 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -11,6 +11,8 @@ #include <util/system.h> #include <cuckoocache.h> + +#include <boost/thread/lock_types.hpp> #include <boost/thread/shared_mutex.hpp> namespace { diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 8afbe9ebed..dba5ce621a 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -106,8 +106,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator std::vector<valtype> vSolutions; whichTypeRet = Solver(scriptPubKey, vSolutions); - switch (whichTypeRet) - { + switch (whichTypeRet) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -173,10 +172,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator // Could not find witnessScript, add to missing sigdata.missing_witness_script = uint256(vSolutions[0]); return false; - - default: - return false; - } + } // no default case, so the compiler can warn about missing cases + assert(false); } static CScript PushAll(const std::vector<valtype>& values) diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 7967c01858..4d882cd1f1 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -45,8 +45,7 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) std::string GetTxnOutputType(TxoutType t) { - switch (t) - { + switch (t) { case TxoutType::NONSTANDARD: return "nonstandard"; case TxoutType::PUBKEY: return "pubkey"; case TxoutType::PUBKEYHASH: return "pubkeyhash"; @@ -182,7 +181,8 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) std::vector<valtype> vSolutions; TxoutType whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TxoutType::PUBKEY) { + switch (whichType) { + case TxoutType::PUBKEY: { CPubKey pubKey(vSolutions[0]); if (!pubKey.IsValid()) return false; @@ -190,26 +190,28 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = PKHash(pubKey); return true; } - else if (whichType == TxoutType::PUBKEYHASH) - { + case TxoutType::PUBKEYHASH: { addressRet = PKHash(uint160(vSolutions[0])); return true; } - else if (whichType == TxoutType::SCRIPTHASH) - { + case TxoutType::SCRIPTHASH: { addressRet = ScriptHash(uint160(vSolutions[0])); return true; - } else if (whichType == TxoutType::WITNESS_V0_KEYHASH) { + } + case TxoutType::WITNESS_V0_KEYHASH: { WitnessV0KeyHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { + } + case TxoutType::WITNESS_V0_SCRIPTHASH: { WitnessV0ScriptHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TxoutType::WITNESS_UNKNOWN || whichType == TxoutType::WITNESS_V1_TAPROOT) { + } + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V1_TAPROOT: { WitnessUnknown unk; unk.version = vSolutions[0][0]; std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); @@ -217,8 +219,13 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = unk; return true; } - // Multisig txns have more than one address... - return false; + case TxoutType::MULTISIG: + // Multisig txns have more than one address... + case TxoutType::NULL_DATA: + case TxoutType::NONSTANDARD: + return false; + } // no default case, so the compiler can warn about missing cases + assert(false); } bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) diff --git a/src/streams.h b/src/streams.h index fce411d3df..e78da31cbc 100644 --- a/src/streams.h +++ b/src/streams.h @@ -6,17 +6,19 @@ #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H -#include <support/allocators/zeroafterfree.h> #include <serialize.h> +#include <span.h> +#include <support/allocators/zeroafterfree.h> #include <algorithm> #include <assert.h> #include <ios> #include <limits> +#include <optional> #include <stdint.h> #include <stdio.h> -#include <string> #include <string.h> +#include <string> #include <utility> #include <vector> @@ -202,14 +204,14 @@ public: class CDataStream { protected: - typedef CSerializeData vector_type; + using vector_type = SerializeData; vector_type vch; - unsigned int nReadPos; + unsigned int nReadPos{0}; int nType; int nVersion; -public: +public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; typedef vector_type::difference_type difference_type; @@ -221,62 +223,22 @@ public: typedef vector_type::reverse_iterator reverse_iterator; explicit CDataStream(int nTypeIn, int nVersionIn) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + : nType{nTypeIn}, + nVersion{nVersionIn} {} - CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + explicit CDataStream(Span<const uint8_t> sp, int nTypeIn, int nVersionIn) + : vch(sp.data(), sp.data() + sp.size()), + nType{nTypeIn}, + nVersion{nVersionIn} {} template <typename... Args> CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + : nType{nTypeIn}, + nVersion{nVersionIn} { - Init(nTypeIn, nVersionIn); ::SerializeMany(*this, std::forward<Args>(args)...); } - void Init(int nTypeIn, int nVersionIn) - { - nReadPos = 0; - nType = nTypeIn; - nVersion = nVersionIn; - } - - CDataStream& operator+=(const CDataStream& b) - { - vch.insert(vch.end(), b.begin(), b.end()); - return *this; - } - - friend CDataStream operator+(const CDataStream& a, const CDataStream& b) - { - CDataStream ret = a; - ret += b; - return (ret); - } - std::string str() const { return (std::string(begin(), end())); @@ -297,12 +259,12 @@ public: const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } + iterator insert(iterator it, const uint8_t x) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const uint8_t x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } - void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) + void insert(iterator it, std::vector<uint8_t>::const_iterator first, std::vector<uint8_t>::const_iterator last) { if (last == first) return; assert(last - first > 0); @@ -373,12 +335,17 @@ public: nReadPos = 0; } - bool Rewind(size_type n) + bool Rewind(std::optional<size_type> n = std::nullopt) { + // Total rewind if no size is passed + if (!n) { + nReadPos = 0; + return true; + } // Rewind by n characters if the buffer hasn't been compacted yet - if (n > nReadPos) + if (*n > nReadPos) return false; - nReadPos -= n; + nReadPos -= *n; return true; } @@ -462,11 +429,6 @@ public: return (*this); } - void GetAndClear(CSerializeData &d) { - d.insert(d.end(), begin(), end()); - clear(); - } - /** * XOR the contents of this stream with a certain key. * diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index c7ed5ef308..418f0ee656 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -42,7 +42,7 @@ struct zero_after_free_allocator : public std::allocator<T> { } }; -// Byte-vector that clears its contents before deletion. -typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; +/** Byte-vector that clears its contents before deletion. */ +using SerializeData = std::vector<uint8_t, zero_after_free_allocator<uint8_t>>; #endif // BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H diff --git a/src/sync.cpp b/src/sync.cpp index acfbe8fe29..a2b62c2286 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -13,8 +13,6 @@ #include <util/strencodings.h> #include <util/threadnames.h> -#include <boost/thread/mutex.hpp> - #include <map> #include <mutex> #include <set> @@ -224,7 +222,6 @@ template void EnterCritical(const char*, const char*, int, Mutex*, bool); template void EnterCritical(const char*, const char*, int, RecursiveMutex*, bool); template void EnterCritical(const char*, const char*, int, std::mutex*, bool); template void EnterCritical(const char*, const char*, int, std::recursive_mutex*, bool); -template void EnterCritical(const char*, const char*, int, boost::mutex*, bool); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 00c4bdc14e..633a95ce96 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -178,7 +178,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -196,7 +196,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -210,7 +210,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -231,14 +231,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { LOCK(cs_main); - block_index = LookupBlockIndex(chainA[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainA_last_header); { LOCK(cs_main); - block_index = LookupBlockIndex(chainB[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainB_last_header); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 736c260eeb..5a98558240 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -42,11 +42,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03614e9b050000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03614e9b050000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); @@ -72,11 +68,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03ce4299050000000100008001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03ce4299050000000100008001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -96,11 +88,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("038fc16b080000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<unsigned char> expected = ParseHex("038fc16b080000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -352,11 +340,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) CDataStream merkleStream(SER_NETWORK, PROTOCOL_VERSION); merkleStream << merkleBlock; - std::vector<unsigned char> vch = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), merkleStream.begin(), merkleStream.end()); } diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 5dbf07b420..21921375b3 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -10,7 +10,6 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> #include <atomic> #include <condition_variable> @@ -148,10 +147,7 @@ typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; static void Correct_Queue_range(std::vector<size_t> range) { auto small_queue = MakeUnique<Correct_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{small_queue->Thread();}); - } + small_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); // Make vChecks here to save on malloc (this test can be slow...) std::vector<FakeCheckCheckCompletion> vChecks; for (const size_t i : range) { @@ -168,8 +164,7 @@ static void Correct_Queue_range(std::vector<size_t> range) BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); } } - tg.interrupt_all(); - tg.join_all(); + small_queue->StopWorkerThreads(); } /** Test that 0 checks is correct @@ -212,11 +207,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1001; ++i) { CCheckQueueControl<FailingCheck> control(fail_queue.get()); @@ -237,18 +228,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) BOOST_REQUIRE(success); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that a block validation which fails does not interfere with // future blocks, ie, the bad state is cleared. BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (auto times = 0; times < 10; ++times) { for (const bool end_fails : {true, false}) { @@ -263,8 +250,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_REQUIRE(r != end_fails); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that unique checks are actually all called individually, rather than @@ -273,11 +259,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { auto queue = MakeUnique<Unique_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); size_t COUNT = 100000; size_t total = COUNT; @@ -300,8 +282,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) } BOOST_REQUIRE(r); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } @@ -313,10 +294,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { auto queue = MakeUnique<Memory_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1000; ++i) { size_t total = i; { @@ -335,8 +313,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) } BOOST_REQUIRE_EQUAL(MemoryCheck::fake_allocated_memory, 0U); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } // Test that a new verification cannot occur until all checks @@ -344,11 +321,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { auto queue = MakeUnique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; bool fails = false; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); std::thread t0([&]() { CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); std::vector<FrozenCleanupCheck> vChecks(1); @@ -367,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) } // Try to get control of the queue a bunch of times for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } { // Unfreeze (we need lock n case of spurious wakeup) @@ -378,9 +352,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) FrozenCleanupCheck::cv.notify_one(); // Wait for control to finish t0.join(); - tg.interrupt_all(); - tg.join_all(); BOOST_REQUIRE(!fails); + queue->StopWorkerThreads(); } @@ -389,11 +362,11 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) { auto queue = MakeUnique<Standard_Queue>(QUEUE_BATCH_SIZE); { - boost::thread_group tg; + std::vector<std::thread> tg; std::atomic<int> nThreads {0}; std::atomic<int> fails {0}; for (size_t i = 0; i < 3; ++i) { - tg.create_thread( + tg.emplace_back( [&]{ CCheckQueueControl<FakeCheck> control(queue.get()); // While sleeping, no other thread should execute to this point @@ -402,11 +375,13 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) fails += observed != nThreads; }); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } BOOST_REQUIRE_EQUAL(fails, 0); } { - boost::thread_group tg; + std::vector<std::thread> tg; std::mutex m; std::condition_variable cv; bool has_lock{false}; @@ -415,7 +390,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) bool done_ack{false}; { std::unique_lock<std::mutex> l(m); - tg.create_thread([&]{ + tg.emplace_back([&]{ CCheckQueueControl<FakeCheck> control(queue.get()); std::unique_lock<std::mutex> ll(m); has_lock = true; @@ -431,7 +406,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.wait(l, [&](){return has_lock;}); bool fails = false; for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } has_tried = true; cv.notify_one(); @@ -441,8 +416,9 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.notify_one(); BOOST_REQUIRE(!fails); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } } } BOOST_AUTO_TEST_SUITE_END() - diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 3a951d28ae..75c7e47e64 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> +#include <boost/thread/lock_types.hpp> +#include <boost/thread/shared_mutex.hpp> #include <cuckoocache.h> #include <deque> #include <random.h> diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index d02c3613ba..e52cd5230c 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <test/util/setup_common.h> #include <util/system.h> +#include <util/getuniquepath.h> #include <boost/test/unit_test.hpp> @@ -52,6 +53,38 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) file >> input_buffer; BOOST_CHECK_EQUAL(input_buffer, "bitcoin"); } + { + // Join an absolute path and a relative path. + fs::path p = fsbridge::AbsPathJoin(tmpfolder, "fs_tests_₿_🏃"); + BOOST_CHECK(p.is_absolute()); + BOOST_CHECK_EQUAL(tmpfile1, p); + } + { + // Join two absolute paths. + fs::path p = fsbridge::AbsPathJoin(tmpfile1, tmpfile2); + BOOST_CHECK(p.is_absolute()); + BOOST_CHECK_EQUAL(tmpfile2, p); + } + { + // Ensure joining with empty paths does not add trailing path components. + BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, "")); + BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, {})); + } + { + fs::path p1 = GetUniquePath(tmpfolder); + fs::path p2 = GetUniquePath(tmpfolder); + fs::path p3 = GetUniquePath(tmpfolder); + + // Ensure that the parent path is always the same. + BOOST_CHECK_EQUAL(tmpfolder, p1.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p2.parent_path()); + BOOST_CHECK_EQUAL(tmpfolder, p3.parent_path()); + + // Ensure that generated paths are actually different. + BOOST_CHECK(p1 != p2); + BOOST_CHECK(p2 != p3); + BOOST_CHECK(p1 != p3); + } } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_SUITE_END()
\ No newline at end of file diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index e703fa39c1..e0715f3e29 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -26,7 +26,7 @@ int64_t ConsumeBanTimeOffset(FuzzedDataProvider& fuzzed_data_provider) noexcept void initialize_banman() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<>(); } FUZZ_TARGET_INIT(banman, initialize_banman) diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 12ef4b203f..8ece94d771 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -36,9 +36,7 @@ bool operator==(const Coin& a, const Coin& b) void initialize_coins_view() { - static const ECCVerifyHandle ecc_verify_handle; - ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + static const auto testing_setup = MakeFuzzingContext<const TestingSetup>(); } FUZZ_TARGET_INIT(coins_view, initialize_coins_view) diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 7950213bd5..71b4b00116 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -17,7 +17,7 @@ void initialize_connman() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<>(); } FUZZ_TARGET_INIT(connman, initialize_connman) diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index 1f122082b2..0e1c44cded 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -45,18 +45,24 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead) assert(ok); }, [&] { + if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) { + return; + } seqnr_payload += 1; aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { aad_pos = 0; + if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) { + return; + } seqnr_aad += 1; } }, [&] { - seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>(); + seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); }, [&] { - seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>(); + seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); }, [&] { is_encrypt = fuzzed_data_provider.ConsumeBool(); diff --git a/src/test/fuzz/data_stream.cpp b/src/test/fuzz/data_stream.cpp index 28fc528ceb..f3b6e6af04 100644 --- a/src/test/fuzz/data_stream.cpp +++ b/src/test/fuzz/data_stream.cpp @@ -13,7 +13,7 @@ void initialize_data_stream_addr_man() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<>(); } FUZZ_TARGET_INIT(data_stream_addr_man, initialize_data_stream_addr_man) diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 74dec6475e..ba5f0c1a75 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -30,8 +30,6 @@ #include <stdint.h> #include <unistd.h> -#include <vector> - #include <test/fuzz/fuzz.h> void initialize_deserialize() @@ -71,7 +69,7 @@ T Deserialize(CDataStream ds) } template <typename T> -void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj, const Optional<int> protocol_version = nullopt) +void DeserializeFromFuzzingInput(FuzzBufferType buffer, T& obj, const Optional<int> protocol_version = nullopt) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); if (protocol_version) { diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index fd87667755..45151258b3 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -13,15 +13,15 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>>& FuzzTargets() +std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>>& FuzzTargets() { - static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>> g_fuzz_targets; + static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize, TypeHidden>> g_fuzz_targets; return g_fuzz_targets; } -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init) +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden) { - const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init)); + const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init), hidden); Assert(it_ins.second); } @@ -31,6 +31,7 @@ void initialize() { if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { + if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } Assert(false); @@ -59,8 +60,7 @@ static bool read_stdin(std::vector<uint8_t>& data) extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { static const auto& test_one_input = *Assert(g_test_one_input); - const std::vector<uint8_t> input(data, data + size); - test_one_input(input); + test_one_input({data, size}); return 0; } diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 52841e069a..4abc52c15a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -5,29 +5,36 @@ #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H +#include <span.h> + #include <cstdint> #include <functional> #include <string_view> -#include <vector> -using TypeTestOneInput = std::function<void(const std::vector<uint8_t>&)>; +using FuzzBufferType = Span<const uint8_t>; + +using TypeTestOneInput = std::function<void(FuzzBufferType)>; using TypeInitialize = std::function<void()>; +using TypeHidden = bool; -void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init); +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init, TypeHidden hidden); -inline void FuzzFrameworkEmptyFun() {} +inline void FuzzFrameworkEmptyInitFun() {} #define FUZZ_TARGET(name) \ - FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyFun) - -#define FUZZ_TARGET_INIT(name, init_fun) \ - void name##_fuzz_target(const std::vector<uint8_t>&); \ - struct name##_Before_Main { \ - name##_Before_Main() \ - { \ - FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun); \ - } \ - } const static g_##name##_before_main; \ - void name##_fuzz_target(const std::vector<uint8_t>& buffer) + FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyInitFun) + +#define FUZZ_TARGET_INIT(name, init_fun) \ + FUZZ_TARGET_INIT_HIDDEN(name, init_fun, false) + +#define FUZZ_TARGET_INIT_HIDDEN(name, init_fun, hidden) \ + void name##_fuzz_target(FuzzBufferType); \ + struct name##_Before_Main { \ + name##_Before_Main() \ + { \ + FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun, hidden); \ + } \ + } const static g_##name##_before_main; \ + void name##_fuzz_target(FuzzBufferType buffer) #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index c428a86631..95597bf082 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -15,7 +15,7 @@ void initialize_load_external_block_file() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<const TestingSetup>(); } FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) @@ -27,5 +27,5 @@ FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) return; } FlatFilePos flat_file_pos; - LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); + ::ChainstateActive().LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); } diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 31b99600ef..21dca4eb05 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -22,7 +22,7 @@ void initialize_net() { - static const BasicTestingSetup basic_testing_setup; + static const auto testing_setup = MakeFuzzingContext<>(CBaseChainParams::MAIN); } FUZZ_TARGET_INIT(net, initialize_net) diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 6e9bb47ff6..a42080eb66 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -9,7 +9,6 @@ #include <cassert> #include <cstdint> -#include <netinet/in.h> #include <vector> FUZZ_TARGET(netaddress) diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 0393491e4b..fff893fb3f 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -16,7 +16,7 @@ void initialize_policy_estimator() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<>(); } FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 8fa52143d8..73242870a0 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -12,7 +12,7 @@ void initialize_policy_estimator_io() { - InitializeFuzzingContext(); + static const auto testing_setup = MakeFuzzingContext<>(); } FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 02beb6eb37..c4348495bf 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -43,7 +43,10 @@ FUZZ_TARGET_INIT(pow, initialize_pow) current_block.nHeight = current_height; } if (fuzzed_data_provider.ConsumeBool()) { - current_block.nTime = fixed_time + current_height * consensus_params.nPowTargetSpacing; + const uint32_t seconds = current_height * consensus_params.nPowTargetSpacing; + if (!AdditionOverflow(fixed_time, seconds)) { + current_block.nTime = fixed_time + seconds; + } } if (fuzzed_data_provider.ConsumeBool()) { current_block.nBits = fixed_bits; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 58637662c5..667ed42447 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -30,45 +30,63 @@ #include <iostream> #include <memory> #include <string> -#include <vector> namespace { const TestingSetup* g_setup; } // namespace +size_t& GetNumMsgTypes() +{ + static size_t g_num_msg_types{0}; + return g_num_msg_types; +} +#define FUZZ_TARGET_MSG(msg_type) \ + struct msg_type##_Count_Before_Main { \ + msg_type##_Count_Before_Main() \ + { \ + ++GetNumMsgTypes(); \ + } \ + } const static g_##msg_type##_count_before_main; \ + FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ + { \ + fuzz_target(buffer, #msg_type); \ + } + void initialize_process_message() { - static TestingSetup setup{ - CBaseChainParams::REGTEST, - { - "-nodebuglogfile", - }, - }; - g_setup = &setup; + Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + static const auto testing_setup = MakeFuzzingContext<const TestingSetup>(); + g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); } SyncWithValidationInterfaceQueue(); } -void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + SetMockTime(1610000000); // any time to successfully reset ibd chainstate.ResetIbd(); + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } - const bool jump_out_of_ibd{fuzzed_data_provider.ConsumeBool()}; - if (jump_out_of_ibd) chainstate.JumpOutOfIbd(); CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); - FillNode(fuzzed_data_provider, p2p_node); - p2p_node.fSuccessfullyConnected = true; + + const bool successfully_connected{true}; + p2p_node.fSuccessfullyConnected = successfully_connected; connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); + + const auto mock_time = ConsumeTime(fuzzed_data_provider); + SetMockTime(mock_time); // fuzzed_data_provider is fully consumed after this call, don't use it CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; @@ -87,27 +105,37 @@ void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO } FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_INIT(process_message_addr, initialize_process_message) { fuzz_target(buffer, "addr"); } -FUZZ_TARGET_INIT(process_message_block, initialize_process_message) { fuzz_target(buffer, "block"); } -FUZZ_TARGET_INIT(process_message_blocktxn, initialize_process_message) { fuzz_target(buffer, "blocktxn"); } -FUZZ_TARGET_INIT(process_message_cmpctblock, initialize_process_message) { fuzz_target(buffer, "cmpctblock"); } -FUZZ_TARGET_INIT(process_message_feefilter, initialize_process_message) { fuzz_target(buffer, "feefilter"); } -FUZZ_TARGET_INIT(process_message_filteradd, initialize_process_message) { fuzz_target(buffer, "filteradd"); } -FUZZ_TARGET_INIT(process_message_filterclear, initialize_process_message) { fuzz_target(buffer, "filterclear"); } -FUZZ_TARGET_INIT(process_message_filterload, initialize_process_message) { fuzz_target(buffer, "filterload"); } -FUZZ_TARGET_INIT(process_message_getaddr, initialize_process_message) { fuzz_target(buffer, "getaddr"); } -FUZZ_TARGET_INIT(process_message_getblocks, initialize_process_message) { fuzz_target(buffer, "getblocks"); } -FUZZ_TARGET_INIT(process_message_getblocktxn, initialize_process_message) { fuzz_target(buffer, "getblocktxn"); } -FUZZ_TARGET_INIT(process_message_getdata, initialize_process_message) { fuzz_target(buffer, "getdata"); } -FUZZ_TARGET_INIT(process_message_getheaders, initialize_process_message) { fuzz_target(buffer, "getheaders"); } -FUZZ_TARGET_INIT(process_message_headers, initialize_process_message) { fuzz_target(buffer, "headers"); } -FUZZ_TARGET_INIT(process_message_inv, initialize_process_message) { fuzz_target(buffer, "inv"); } -FUZZ_TARGET_INIT(process_message_mempool, initialize_process_message) { fuzz_target(buffer, "mempool"); } -FUZZ_TARGET_INIT(process_message_notfound, initialize_process_message) { fuzz_target(buffer, "notfound"); } -FUZZ_TARGET_INIT(process_message_ping, initialize_process_message) { fuzz_target(buffer, "ping"); } -FUZZ_TARGET_INIT(process_message_pong, initialize_process_message) { fuzz_target(buffer, "pong"); } -FUZZ_TARGET_INIT(process_message_sendcmpct, initialize_process_message) { fuzz_target(buffer, "sendcmpct"); } -FUZZ_TARGET_INIT(process_message_sendheaders, initialize_process_message) { fuzz_target(buffer, "sendheaders"); } -FUZZ_TARGET_INIT(process_message_tx, initialize_process_message) { fuzz_target(buffer, "tx"); } -FUZZ_TARGET_INIT(process_message_verack, initialize_process_message) { fuzz_target(buffer, "verack"); } -FUZZ_TARGET_INIT(process_message_version, initialize_process_message) { fuzz_target(buffer, "version"); } +FUZZ_TARGET_MSG(addr); +FUZZ_TARGET_MSG(addrv2); +FUZZ_TARGET_MSG(block); +FUZZ_TARGET_MSG(blocktxn); +FUZZ_TARGET_MSG(cfcheckpt); +FUZZ_TARGET_MSG(cfheaders); +FUZZ_TARGET_MSG(cfilter); +FUZZ_TARGET_MSG(cmpctblock); +FUZZ_TARGET_MSG(feefilter); +FUZZ_TARGET_MSG(filteradd); +FUZZ_TARGET_MSG(filterclear); +FUZZ_TARGET_MSG(filterload); +FUZZ_TARGET_MSG(getaddr); +FUZZ_TARGET_MSG(getblocks); +FUZZ_TARGET_MSG(getblocktxn); +FUZZ_TARGET_MSG(getcfcheckpt); +FUZZ_TARGET_MSG(getcfheaders); +FUZZ_TARGET_MSG(getcfilters); +FUZZ_TARGET_MSG(getdata); +FUZZ_TARGET_MSG(getheaders); +FUZZ_TARGET_MSG(headers); +FUZZ_TARGET_MSG(inv); +FUZZ_TARGET_MSG(mempool); +FUZZ_TARGET_MSG(merkleblock); +FUZZ_TARGET_MSG(notfound); +FUZZ_TARGET_MSG(ping); +FUZZ_TARGET_MSG(pong); +FUZZ_TARGET_MSG(sendaddrv2); +FUZZ_TARGET_MSG(sendcmpct); +FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(tx); +FUZZ_TARGET_MSG(verack); +FUZZ_TARGET_MSG(version); +FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index db52da2f7e..810f0aac92 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -17,18 +17,14 @@ #include <validation.h> #include <validationinterface.h> +namespace { const TestingSetup* g_setup; +} // namespace void initialize_process_messages() { - static TestingSetup setup{ - CBaseChainParams::REGTEST, - { - "-nodebuglogfile", - }, - }; - g_setup = &setup; - + static const auto testing_setup = MakeFuzzingContext<const TestingSetup>(); + g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { MineBlock(g_setup->m_node, CScript() << OP_TRUE); } @@ -41,28 +37,30 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + SetMockTime(1610000000); // any time to successfully reset ibd chainstate.ResetIbd(); - std::vector<CNode*> peers; - bool jump_out_of_ibd{false}; + std::vector<CNode*> peers; const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); for (int i = 0; i < num_peers_to_add; ++i) { peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); - FillNode(fuzzed_data_provider, p2p_node); - p2p_node.fSuccessfullyConnected = true; + const bool successfully_connected{true}; + p2p_node.fSuccessfullyConnected = successfully_connected; p2p_node.fPauseSend = false; g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); connman.AddTestNode(p2p_node); } while (fuzzed_data_provider.ConsumeBool()) { - if (!jump_out_of_ibd) jump_out_of_ibd = fuzzed_data_provider.ConsumeBool(); - if (jump_out_of_ibd && chainstate.IsInitialBlockDownload()) chainstate.JumpOutOfIbd(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + const auto mock_time = ConsumeTime(fuzzed_data_provider); + SetMockTime(mock_time); + CSerializedNetMsg net_msg; net_msg.m_type = random_message_type; net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index d883426c81..7fadf36f98 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -154,13 +154,13 @@ FUZZ_TARGET_INIT(script, initialize_script) { WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_1.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_2.length = witness_unknown_program_2.size(); std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 2091ad5d91..8d9a939dab 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -28,12 +28,12 @@ // // (normal build) // $ mkdir dump -// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot --dumptests; done +// $ for N in $(seq 1 10); do TEST_DUMP_DIR=dump test/functional/feature_taproot.py --dumptests; done // $ ... // -// (fuzz test build) +// (libFuzzer build) // $ mkdir dump-min -// $ ./src/test/fuzz/script_assets_test_minimizer -merge=1 dump-min/ dump/ +// $ FUZZ=script_assets_test_minimizer ./src/test/fuzz/fuzz -merge=1 -use_value_profile=1 dump-min/ dump/ // $ (echo -en '[\n'; cat dump-min/* | head -c -2; echo -en '\n]') >script_assets_test.json namespace { @@ -190,7 +190,7 @@ ECCVerifyHandle handle; } // namespace -FUZZ_TARGET(script_assets_test_minimizer) +FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, FuzzFrameworkEmptyInitFun, /* hidden */ true) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 541322d484..83effec064 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -17,7 +17,7 @@ void initialize_signet() { - InitializeFuzzingContext(CBaseChainParams::SIGNET); + static const auto testing_setup = MakeFuzzingContext<>(CBaseChainParams::SIGNET); } FUZZ_TARGET_INIT(signet, initialize_signet) diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 282a2cd8ca..d6bbb17f63 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -78,7 +78,9 @@ FUZZ_TARGET(string) } (void)SanitizeString(random_string_1); (void)SanitizeString(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)); +#ifndef WIN32 (void)ShellEscape(random_string_1); +#endif // WIN32 int port_out; std::string host_out; SplitHostPort(random_string_1, port_out, host_out); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 47b38b6d23..3621702e45 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -54,7 +54,7 @@ FUZZ_TARGET(system) if (args_manager.GetArgFlags(argument_name) != nullopt) { return; } - args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>() & ~ArgsManager::COMMAND, options_category); }, [&] { // Avoid hitting: diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp new file mode 100644 index 0000000000..0a541e4186 --- /dev/null +++ b/src/test/fuzz/util.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util.h> +#include <version.h> + +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept +{ + const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); + const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); + const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); + const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + + node.nServices = remote_services; + node.m_permissionFlags = permission_flags; + if (init_version) { + node.nVersion = version; + node.SetCommonVersion(std::min(version, PROTOCOL_VERSION)); + } + if (node.m_tx_relay != nullptr) { + LOCK(node.m_tx_relay->cs_filter); + node.m_tx_relay->fRelayTxes = filter_txs; + } +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index b7cf395e76..7a2dcfe84a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -27,6 +27,7 @@ #include <txmempool.h> #include <uint256.h> #include <util/time.h> +#include <util/vector.h> #include <version.h> #include <algorithm> @@ -60,7 +61,7 @@ void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) [[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { - return {ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; + return CDataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; } [[nodiscard]] inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept @@ -195,7 +196,7 @@ template <typename WeakEnumType, size_t size> }, [&] { WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); @@ -268,8 +269,8 @@ inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcep const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); CNetAddr net_addr; if (network == Network::NET_IPV4) { - const in_addr v4_addr = { - .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); net_addr = CNetAddr{v4_addr}; } else if (network == Network::NET_IPV6) { if (fuzzed_data_provider.remaining_bytes() >= 16) { @@ -319,28 +320,21 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N return CNode{node_id, local_services, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion}; } } -inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = nullopt) { return ConsumeNode<true>(fdp, node_id_in); } +inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } -inline void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, const std::optional<int32_t>& version_in = std::nullopt) noexcept -{ - const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); - const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); - const int32_t version = version_in.value_or(fuzzed_data_provider.ConsumeIntegral<int32_t>()); - const bool filter_txs = fuzzed_data_provider.ConsumeBool(); - - node.nServices = remote_services; - node.m_permissionFlags = permission_flags; - node.nVersion = version; - node.SetCommonVersion(version); - if (node.m_tx_relay != nullptr) { - LOCK(node.m_tx_relay->cs_filter); - node.m_tx_relay->fRelayTxes = filter_txs; - } -} +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept; -inline void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) +template <class T = const BasicTestingSetup> +std::unique_ptr<T> MakeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST, const std::vector<const char*>& extra_args = {}) { - static const BasicTestingSetup basic_testing_setup{chain_name, {"-nodebuglogfile"}}; + // Prepend default arguments for fuzzing + const std::vector<const char*> arguments = Cat( + { + "-nodebuglogfile", + }, + extra_args); + + return MakeUnique<T>(chain_name, arguments); } class FuzzedFileProvider diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 2e5a7549b7..d57c000b92 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -7,9 +7,11 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> +#include <functional> #include <mutex> +#include <thread> +#include <vector> BOOST_AUTO_TEST_SUITE(scheduler_tests) @@ -68,16 +70,16 @@ BOOST_AUTO_TEST_CASE(manythreads) BOOST_CHECK(last > now); // As soon as these are created they will start running and servicing the queue - boost::thread_group microThreads; + std::vector<std::thread> microThreads; for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); UninterruptibleSleep(std::chrono::microseconds{600}); now = std::chrono::system_clock::now(); // More threads and more tasks: for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); for (int i = 0; i < 100; i++) { std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); @@ -90,7 +92,10 @@ BOOST_AUTO_TEST_CASE(manythreads) // Drain the task queue then exit threads microTasks.StopWhenDrained(); - microThreads.join_all(); // ... wait until all the threads are done + // wait until all the threads are done + for (auto& thread: microThreads) { + if (thread.joinable()) thread.join(); + } int counterSum = 0; for (int i = 0; i < 10; i++) { @@ -130,9 +135,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // if the queues only permit execution of one task at once then // the extra threads should effectively be doing nothing // if they don't we'll get out of order behaviour - boost::thread_group threads; + std::vector<std::thread> threads; for (int i = 0; i < 5; ++i) { - threads.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + threads.emplace_back(std::bind(&CScheduler::serviceQueue, &scheduler)); } // these are not atomic, if SinglethreadedSchedulerClient prevents @@ -156,7 +161,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // finish up scheduler.StopWhenDrained(); - threads.join_all(); + for (auto& thread: threads) { + if (thread.joinable()) thread.join(); + } BOOST_CHECK_EQUAL(counter1, 100); BOOST_CHECK_EQUAL(counter2, 100); diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 366385b619..4dc0dd5f51 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -107,6 +107,22 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(scriptHash)); + // TxoutType::WITNESS_V1_TAPROOT + s.clear(); + s << OP_1 << ToByteVector(uint256::ZERO); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V1_TAPROOT); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{1}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ZERO)); + + // TxoutType::WITNESS_UNKNOWN + s.clear(); + s << OP_16 << ToByteVector(uint256::ONE); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_UNKNOWN); + BOOST_CHECK_EQUAL(solutions.size(), 2U); + BOOST_CHECK(solutions[0] == std::vector<unsigned char>{16}); + BOOST_CHECK(solutions[1] == ToByteVector(uint256::ONE)); + // TxoutType::NONSTANDARD s.clear(); s << OP_9 << OP_ADD << OP_11 << OP_EQUAL; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index f625b67c2a..f77cda7ba2 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.insert(ss.end(), c); BOOST_CHECK_EQUAL(ss.size(), 6U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); BOOST_CHECK_EQUAL(ss[5], c); ss.insert(ss.begin()+2, c); @@ -334,19 +334,14 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.erase(ss.begin()+ss.size()-1); BOOST_CHECK_EQUAL(ss.size(), 5U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); ss.erase(ss.begin()+1); BOOST_CHECK_EQUAL(ss.size(), 4U); BOOST_CHECK_EQUAL(ss[0], 0); BOOST_CHECK_EQUAL(ss[1], 1); BOOST_CHECK_EQUAL(ss[2], 2); - BOOST_CHECK_EQUAL(ss[3], (char)0xff); - - // Make sure GetAndClear does the right thing: - CSerializeData d; - ss.GetAndClear(d); - BOOST_CHECK_EQUAL(ss.size(), 0U); + BOOST_CHECK_EQUAL(ss[3], 0xff); } BOOST_AUTO_TEST_CASE(class_methods) diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 8f5ee1e8e2..3079c9ff29 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { - std::vector<char> in; + std::vector<uint8_t> in; std::vector<char> expected_xor; std::vector<unsigned char> key; CDataStream ds(in, 0, 0); diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 71275f69d9..3e4d1dac9e 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -6,7 +6,6 @@ #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/mutex.hpp> #include <mutex> @@ -110,11 +109,6 @@ BOOST_AUTO_TEST_CASE(double_lock_mutex) TestDoubleLock<Mutex>(true /* should throw */); } -BOOST_AUTO_TEST_CASE(double_lock_boost_mutex) -{ - TestDoubleLock<boost::mutex>(true /* should throw */); -} - BOOST_AUTO_TEST_CASE(double_lock_recursive_mutex) { TestDoubleLock<RecursiveMutex>(false /* should not throw */); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 5b35ed6976..1eb30c68d6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -24,6 +24,7 @@ #include <util/strencodings.h> #include <validation.h> +#include <functional> #include <map> #include <string> @@ -428,12 +429,10 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); - boost::thread_group threadGroup; CCheckQueue<CScriptCheck> scriptcheckqueue(128); CCheckQueueControl<CScriptCheck> control(&scriptcheckqueue); - for (int i=0; i<20; i++) - threadGroup.create_thread(std::bind(&CCheckQueue<CScriptCheck>::Thread, std::ref(scriptcheckqueue))); + scriptcheckqueue.StartWorkerThreads(20); std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { @@ -455,9 +454,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) bool controlCheck = control.Wait(); assert(controlCheck); - - threadGroup.interrupt_all(); - threadGroup.join_all(); + scriptcheckqueue.StopWorkerThreads(); } SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutableTransaction& input2, const CTransactionRef tx) diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 738f414cd0..b9f3f8c955 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -131,7 +131,7 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. m_node.scheduler = MakeUnique<CScheduler>(); - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); + m_node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -143,17 +143,14 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; - for (int i = 0; i < script_check_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_check_threads); g_parallel_script_checks = true; } ChainTestingSetup::~ChainTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); m_node.connman.reset(); @@ -186,7 +183,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index dff8cf825e..331c1235cb 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -17,8 +17,7 @@ #include <util/string.h> #include <type_traits> - -#include <boost/thread/thread.hpp> +#include <vector> /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -88,7 +87,6 @@ private: * initialization behaviour. */ struct ChainTestingSetup : public BasicTestingSetup { - boost::thread_group threadGroup; explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 4133f2623b..dccbf00d6f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -13,6 +13,7 @@ #include <test/util/setup_common.h> #include <test/util/str.h> #include <uint256.h> +#include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/spanparsing.h> @@ -1816,7 +1817,7 @@ BOOST_AUTO_TEST_CASE(test_DirIsWritable) BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), true); // Should not be able to write to a non-existent dir. - tmpdirname = tmpdirname / fs::unique_path(); + tmpdirname = GetUniquePath(tmpdirname); BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), false); fs::create_directory(tmpdirname); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index ea17cb50f1..ec45d9a434 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -95,8 +95,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { - LOCK(cs_main); // For LookupBlockIndex - GenerateCoinbaseCommitment(*pblock, LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); + LOCK(cs_main); // For g_chainman.m_blockman.LookupBlockIndex + GenerateCoinbaseCommitment(*pblock, g_chainman.m_blockman.LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 208794a4e5..605c77fc3a 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -16,10 +16,11 @@ #include <util/system.h> #include <util/time.h> -#include <vector> #include <deque> +#include <functional> #include <set> #include <stdlib.h> +#include <vector> #include <boost/signals2/signal.hpp> #include <boost/algorithm/string/split.hpp> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 9ae7b921b3..9fa7b4e251 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -9,14 +9,14 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <optional.h> -#include <validation.h> -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <policy/settings.h> #include <reverse_iterator.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <util/time.h> +#include <validation.h> #include <validationinterface.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, @@ -396,7 +396,10 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} + m_total_fee += entry.GetFee(); + if (minerPolicyEstimator) { + minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + } vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -432,6 +435,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) vTxHashes.clear(); totalTxSize -= it->GetTxSize(); + m_total_fee -= it->GetFee(); cachedInnerUsage -= it->DynamicMemoryUsage(); cachedInnerUsage -= memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); mapTx.erase(it); @@ -590,6 +594,7 @@ void CTxMemPool::_clear() mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; + m_total_fee = 0; cachedInnerUsage = 0; lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = false; @@ -618,19 +623,22 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(m_check_ratio) >= 1) return; + AssertLockHeld(::cs_main); LOCK(cs); LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; + CAmount check_total_fee{0}; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); - const int64_t spendheight = GetSpendHeight(mempoolDuplicate); + const int64_t spendheight = g_chainman.m_blockman.GetSpendHeight(mempoolDuplicate); std::list<const CTxMemPoolEntry*> waitingOnDependants; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; checkTotal += it->GetTxSize(); + check_total_fee += it->GetFee(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); @@ -725,6 +733,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const } assert(totalTxSize == checkTotal); + assert(m_total_fee == check_total_fee); assert(innerUsage == cachedInnerUsage); } diff --git a/src/txmempool.h b/src/txmempool.h index 9a1aa9bc2b..c0df33fe13 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -19,8 +19,8 @@ #include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> -#include <sync.h> #include <random.h> +#include <sync.h> #include <util/hasher.h> #include <boost/multi_index_container.hpp> @@ -478,8 +478,9 @@ private: std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; - uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. - uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) + uint64_t totalTxSize GUARDED_BY(cs); //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. + CAmount m_total_fee GUARDED_BY(cs); //!< sum of all mempool tx's fees (NOT modified fee) + uint64_t cachedInnerUsage GUARDED_BY(cs); //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; @@ -602,7 +603,7 @@ public: * all inputs are in the mapNextTx array). If sanity-checking is turned off, * check does nothing. */ - void check(const CCoinsViewCache *pcoins) const; + void check(const CCoinsViewCache *pcoins) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of @@ -724,6 +725,12 @@ public: return totalTxSize; } + CAmount GetTotalFee() const EXCLUSIVE_LOCKS_REQUIRED(cs) + { + AssertLockHeld(cs); + return m_total_fee; + } + bool exists(const GenTxid& gtxid) const { LOCK(cs); diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp new file mode 100644 index 0000000000..9839d2f624 --- /dev/null +++ b/src/util/getuniquepath.cpp @@ -0,0 +1,10 @@ +#include <random.h> +#include <fs.h> +#include <util/strencodings.h> + +fs::path GetUniquePath(const fs::path& base) +{ + FastRandomContext rnd; + fs::path tmpFile = base / HexStr(rnd.randbytes(8)); + return tmpFile; +}
\ No newline at end of file diff --git a/src/util/getuniquepath.h b/src/util/getuniquepath.h new file mode 100644 index 0000000000..e0c6147876 --- /dev/null +++ b/src/util/getuniquepath.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GETUNIQUEPATH_H +#define BITCOIN_UTIL_GETUNIQUEPATH_H + +#include <fs.h> + +/** + * Helper function for getting a unique path + * + * @param[in] base Base path + * @returns base joined with a random 8-character long string. + * @post Returned path is unique with high probability. + */ +fs::path GetUniquePath(const fs::path& base); + +#endif // BITCOIN_UTIL_GETUNIQUEPATH_H
\ No newline at end of file diff --git a/src/util/macros.h b/src/util/macros.h index 36ea87c0fe..0887c80fd7 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -8,4 +8,11 @@ #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) +/** + * Converts the parameter X to a string after macro replacement on X has been performed. + * Don't merge these into one macro! + */ +#define STRINGIZE(X) DO_STRINGIZE(X) +#define DO_STRINGIZE(X) #X + #endif // BITCOIN_UTIL_MACROS_H diff --git a/src/util/system.cpp b/src/util/system.cpp index 917d37ccc7..9a2e719bbc 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,7 +3,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <sync.h> #include <util/system.h> #ifdef HAVE_BOOST_PROCESS @@ -11,6 +10,9 @@ #endif // HAVE_BOOST_PROCESS #include <chainparamsbase.h> +#include <sync.h> +#include <util/check.h> +#include <util/getuniquepath.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -123,7 +125,7 @@ void ReleaseDirectoryLocks() bool DirIsWritable(const fs::path& directory) { - fs::path tmpFile = directory / fs::unique_path(); + fs::path tmpFile = GetUniquePath(directory); FILE* file = fsbridge::fopen(tmpFile, "a"); if (!file) return false; @@ -310,8 +312,22 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin key[0] = '-'; #endif - if (key[0] != '-') + if (key[0] != '-') { + if (!m_accept_any_command && m_command.empty()) { + // The first non-dash arg is a registered command + Optional<unsigned int> flags = GetArgFlags(key); + if (!flags || !(*flags & ArgsManager::COMMAND)) { + error = strprintf("Invalid command '%s'", argv[i]); + return false; + } + } + m_command.push_back(key); + while (++i < argc) { + // The remaining args are command args + m_command.push_back(argv[i]); + } break; + } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') @@ -359,6 +375,26 @@ Optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) const return nullopt; } +std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const +{ + Command ret; + LOCK(cs_args); + auto it = m_command.begin(); + if (it == m_command.end()) { + // No command was passed + return std::nullopt; + } + if (!m_accept_any_command) { + // The registered command + ret.command = *(it++); + } + while (it != m_command.end()) { + // The unregistered command and args (if any) + ret.args.push_back(*(it++)); + } + return ret; +} + std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; @@ -398,7 +434,7 @@ bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const } if (filepath) { std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); - *filepath = fs::absolute(temp ? settings + ".tmp" : settings, GetDataDir(/* net_specific= */ true)); + *filepath = fsbridge::AbsPathJoin(GetDataDir(/* net_specific= */ true), temp ? settings + ".tmp" : settings); } return true; } @@ -504,8 +540,22 @@ void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strV m_settings.forced_settings[SettingName(strArg)] = strValue; } +void ArgsManager::AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat) +{ + Assert(cmd.find('=') == std::string::npos); + Assert(cmd.at(0) != '-'); + + LOCK(cs_args); + m_accept_any_command = false; // latch to false + std::map<std::string, Arg>& arg_map = m_available_args[cat]; + auto ret = arg_map.emplace(cmd, Arg{"", help, ArgsManager::COMMAND}); + Assert(ret.second); // Fail on duplicate commands +} + void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) { + Assert((flags & ArgsManager::COMMAND) == 0); // use AddCommand + // Split arg name from its help param size_t eq_index = name.find('='); if (eq_index == std::string::npos) { @@ -1311,7 +1361,7 @@ fs::path AbsPathForConfigVal(const fs::path& path, bool net_specific) if (path.is_absolute()) { return path; } - return fs::absolute(path, GetDataDir(net_specific)); + return fsbridge::AbsPathJoin(GetDataDir(net_specific), path); } void ScheduleBatchPriority() diff --git a/src/util/system.h b/src/util/system.h index 010fc5b49f..5959bc4196 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -35,8 +35,6 @@ #include <utility> #include <vector> -#include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted - class UniValue; // Application startup time (used for uptime calculation) @@ -168,7 +166,7 @@ struct SectionInfo class ArgsManager { public: - enum Flags { + enum Flags : uint32_t { // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 ALLOW_BOOL = 0x01, ALLOW_INT = 0x02, @@ -183,6 +181,7 @@ public: NETWORK_ONLY = 0x200, // This argument's value is sensitive (such as a password). SENSITIVE = 0x400, + COMMAND = 0x800, }; protected: @@ -195,9 +194,11 @@ protected: mutable RecursiveMutex cs_args; util::Settings m_settings GUARDED_BY(cs_args); + std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); + bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); [[nodiscard]] bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); @@ -248,6 +249,20 @@ public: */ const std::list<SectionInfo> GetUnrecognizedSections() const; + struct Command { + /** The command (if one has been registered with AddCommand), or empty */ + std::string command; + /** + * If command is non-empty: Any args that followed it + * If command is empty: The unregistered command and any args that followed it + */ + std::vector<std::string> args; + }; + /** + * Get the command and command args (returns std::nullopt if no command provided) + */ + std::optional<const Command> GetCommand() const; + /** * Return a vector of strings of the given argument * @@ -334,6 +349,11 @@ public: void AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat); /** + * Add subcommand + */ + void AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat); + + /** * Add many hidden arguments */ void AddHiddenArgs(const std::vector<std::string>& args); @@ -450,11 +470,6 @@ template <typename Callable> void TraceThread(const char* name, Callable func) func(); LogPrintf("%s thread exit\n", name); } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread interrupt\n", name); - throw; - } catch (const std::exception& e) { PrintExceptionContinue(&e, name); throw; diff --git a/src/util/time.cpp b/src/util/time.cpp index 4aed9f60b0..295806c54a 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -10,6 +10,8 @@ #include <compat.h> #include <util/time.h> +#include <util/check.h> + #include <atomic> #include <boost/date_time/posix_time/posix_time.hpp> #include <ctime> @@ -19,7 +21,7 @@ void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } -static std::atomic<int64_t> nMockTime(0); //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For testing int64_t GetTime() { @@ -47,6 +49,7 @@ template std::chrono::microseconds GetTime(); void SetMockTime(int64_t nMockTimeIn) { + Assert(nMockTimeIn >= 0); nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } diff --git a/src/util/trace.h b/src/util/trace.h new file mode 100644 index 0000000000..9c92cb10e7 --- /dev/null +++ b/src/util/trace.h @@ -0,0 +1,45 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_TRACE_H +#define BITCOIN_UTIL_TRACE_H + +#ifdef ENABLE_TRACING + +#include <sys/sdt.h> + +#define TRACE(context, event) DTRACE_PROBE(context, event) +#define TRACE1(context, event, a) DTRACE_PROBE1(context, event, a) +#define TRACE2(context, event, a, b) DTRACE_PROBE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) DTRACE_PROBE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) DTRACE_PROBE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) DTRACE_PROBE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) DTRACE_PROBE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) + +#else + +#define TRACE(context, event) +#define TRACE1(context, event, a) +#define TRACE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) + +#endif + + +#endif /* BITCOIN_UTIL_TRACE_H */ diff --git a/src/validation.cpp b/src/validation.cpp index 4f7c95b2ad..8f7d36bfd3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -167,17 +167,19 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -CBlockIndex* LookupBlockIndex(const uint256& hash) +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); - return it == g_chainman.BlockIndex().end() ? nullptr : it->second; + assert(std::addressof(g_chainman.BlockIndex()) == std::addressof(m_block_index)); + BlockMap::const_iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : it->second; } -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { @@ -404,36 +406,41 @@ static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransact LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } -// Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool -// were somehow broken and returning the wrong scriptPubKeys -static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, - unsigned int flags, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +/** +* Checks to avoid mempool polluting consensus critical paths since cached +* signature and script validity results will be reused if we validate this +* transaction again during block validation. +* */ +static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, + const CCoinsViewCache& view, const CTxMemPool& pool, + unsigned int flags, PrecomputedTransactionData& txdata) + EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ AssertLockHeld(cs_main); - - // pool.cs should be locked already, but go ahead and re-take the lock here - // to enforce that mempool doesn't change between when we check the view - // and when we actually call through to CheckInputScripts - LOCK(pool.cs); + AssertLockHeld(pool.cs); assert(!tx.IsCoinBase()); for (const CTxIn& txin : tx.vin) { const Coin& coin = view.AccessCoin(txin.prevout); - // AcceptToMemoryPoolWorker has already checked that the coins are - // available, so this shouldn't fail. If the inputs are not available - // here then return false. + // This coin was checked in PreChecks and MemPoolAccept + // has been holding cs_main since then. + Assume(!coin.IsSpent()); if (coin.IsSpent()) return false; - // Check equivalence for available inputs. + // If the Coin is available, there are 2 possibilities: + // it is available in our current ChainstateActive UTXO set, + // or it's a UTXO provided by a transaction in our mempool. + // Ensure the scriptPubKeys in Coins from CoinsView are correct. const CTransactionRef& txFrom = pool.get(txin.prevout.hash); if (txFrom) { assert(txFrom->GetHash() == txin.prevout.hash); assert(txFrom->vout.size() > txin.prevout.n); assert(txFrom->vout[txin.prevout.n] == coin.out); } else { - const Coin& coinFromDisk = ::ChainstateActive().CoinsTip().AccessCoin(txin.prevout); - assert(!coinFromDisk.IsSpent()); - assert(coinFromDisk.out == coin.out); + const Coin& coinFromUTXOSet = ::ChainstateActive().CoinsTip().AccessCoin(txin.prevout); + assert(!coinFromUTXOSet.IsSpent()); + assert(coinFromUTXOSet.out == coin.out); } } @@ -502,13 +509,13 @@ private: // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size @@ -516,7 +523,7 @@ private: bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. - bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) + bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) { CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { @@ -677,7 +684,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); CAmount nFees = 0; - if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { + if (!Consensus::CheckTxInputs(tx, state, m_view, g_chainman.m_blockman.GetSpendHeight(m_view), nFees)) { return false; // state filled in by CheckTxInputs } @@ -1257,8 +1264,8 @@ void CoinsViews::InitCache() } CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash) - : m_blockman(blockman), - m_mempool(mempool), + : m_mempool(mempool), + m_blockman(blockman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1405,9 +1412,10 @@ bool CScriptCheck::operator()() { return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *txdata), &error); } -int GetSpendHeight(const CCoinsViewCache& inputs) +int BlockManager::GetSpendHeight(const CCoinsViewCache& inputs) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindexPrev = LookupBlockIndex(inputs.GetBestBlock()); return pindexPrev->nHeight + 1; } @@ -1758,9 +1766,14 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationSt static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck(int worker_num) { - util::ThreadRename(strprintf("scriptch.%i", worker_num)); - scriptcheckqueue.Thread(); +void StartScriptCheckWorkerThreads(int threads_num) +{ + scriptcheckqueue.StartWorkerThreads(threads_num); +} + +void StopScriptCheckWorkerThreads() +{ + scriptcheckqueue.StopWorkerThreads(); } VersionBitsCache versionbitscache GUARDED_BY(cs_main); @@ -2757,7 +2770,7 @@ static SynchronizationState GetSynchronizationState(bool init) return SynchronizationState::INIT_DOWNLOAD; } -static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { +static bool NotifyHeaderTip(CChainState& chainstate) LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex* pindexHeaderOld = nullptr; @@ -2768,7 +2781,8 @@ static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = ::ChainstateActive().IsInitialBlockDownload(); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + fInitialBlockDownload = chainstate.IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } @@ -2885,10 +2899,6 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar return true; } -bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { - return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); -} - bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { @@ -3388,14 +3398,14 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -//! Returns last CBlockIndex* that is a checkpoint -static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindex = LookupBlockIndex(hash); if (pindex) { return pindex; @@ -3413,7 +3423,7 @@ static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOC * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, BlockManager& blockman, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { assert(pindexPrev != nullptr); const int nHeight = pindexPrev->nHeight + 1; @@ -3428,7 +3438,8 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -3548,11 +3559,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator miSelf = m_block_index.find(hash); - CBlockIndex *pindex = nullptr; if (hash != chainparams.GetConsensus().hashGenesisBlock) { if (miSelf != m_block_index.end()) { // Block header is already known. - pindex = miSelf->second; + CBlockIndex* pindex = miSelf->second; if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { @@ -3579,7 +3589,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS LogPrintf("ERROR: %s: prev block invalid\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); /* Determine if this block descends from any block which has been found @@ -3621,8 +3631,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } } } - if (pindex == nullptr) - pindex = AddToBlockIndex(block); + CBlockIndex* pindex = AddToBlockIndex(block); if (ppindex) *ppindex = pindex; @@ -3633,6 +3642,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Exposed wrapper for AcceptBlockHeader bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); AssertLockNotHeld(cs_main); { LOCK(cs_main); @@ -3640,7 +3650,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); - ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); + ActiveChainstate().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { return false; @@ -3650,8 +3660,8 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } } } - if (NotifyHeaderTip()) { - if (::ChainstateActive().IsInitialBlockDownload() && ppindex && *ppindex) { + if (NotifyHeaderTip(ActiveChainstate())) { + if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", (*ppindex)->nHeight, 100.0/((*ppindex)->nHeight+(GetAdjustedTime() - (*ppindex)->GetBlockTime()) / Params().GetConsensus().nPowTargetSpacing) * (*ppindex)->nHeight); } } @@ -3763,6 +3773,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { AssertLockNotHeld(cs_main); + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); { CBlockIndex *pindex = nullptr; @@ -3778,7 +3789,7 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s bool ret = CheckBlock(*pblock, state, chainparams.GetConsensus()); if (ret) { // Store to disk - ret = ::ChainstateActive().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); + ret = ActiveChainstate().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } if (!ret) { GetMainSignals().BlockChecked(*pblock, state); @@ -3786,20 +3797,27 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s } } - NotifyHeaderTip(); + NotifyHeaderTip(ActiveChainstate()); BlockValidationState state; // Only used to report errors, not invalidity - ignore it - if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) + if (!ActiveChainstate().ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); return true; } -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW, + bool fCheckMerkleRoot) { AssertLockHeld(cs_main); - assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); - CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + assert(pindexPrev && pindexPrev == chainstate.m_chain.Tip()); + CCoinsViewCache viewNew(&chainstate.CoinsTip()); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -3807,13 +3825,14 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar indexDummy.phashBlock = &block_hash; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + assert(std::addressof(g_chainman.m_blockman) == std::addressof(chainstate.m_blockman)); + if (!ContextualCheckBlockHeader(block, state, chainstate.m_blockman, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, state.ToString()); if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -4159,7 +4178,7 @@ bool CChainState::LoadChainTip(const CChainParams& chainparams) } // Load pointer to end of best chain - CBlockIndex* pindex = LookupBlockIndex(coins_cache.GetBestBlock()); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(coins_cache.GetBestBlock()); if (!pindex) { return false; } @@ -4589,7 +4608,7 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return ::ChainstateActive().LoadGenesisBlock(chainparams); } -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) +void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4638,7 +4657,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(block.hashPrevBlock)) { + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + if (hash != chainparams.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) @@ -4647,10 +4667,12 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } // process in case the block isn't known yet - CBlockIndex* pindex = LookupBlockIndex(hash); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; - if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { nLoaded++; } if (state.IsError()) { @@ -4664,12 +4686,14 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); // Recursively process earlier encountered successors of this block std::deque<uint256> queue; @@ -4687,7 +4711,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi head.ToString()); LOCK(cs_main); BlockValidationState dummy; - if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -4695,7 +4720,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } range.first++; mapBlocksUnknownParent.erase(it); - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); } } } catch (const std::exception& e) { @@ -5133,6 +5159,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin } Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + LOCK(::cs_main); if (m_active_chainstate != nullptr) { // If a snapshot chainstate exists, it will always be our active. return m_active_chainstate->m_from_snapshot_blockhash; @@ -5142,6 +5169,7 @@ Optional<uint256> ChainstateManager::SnapshotBlockhash() const { std::vector<CChainState*> ChainstateManager::GetAll() { + LOCK(::cs_main); std::vector<CChainState*> out; if (!IsSnapshotValidated() && m_ibd_chainstate) { @@ -5179,17 +5207,20 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const CChainState& ChainstateManager::ActiveChainstate() const { + LOCK(::cs_main); assert(m_active_chainstate); return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const { + LOCK(::cs_main); return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } CChainState& ChainstateManager::ValidatedChainstate() const { + LOCK(::cs_main); if (m_snapshot_chainstate && IsSnapshotValidated()) { return *m_snapshot_chainstate.get(); } @@ -5199,6 +5230,7 @@ CChainState& ChainstateManager::ValidatedChainstate() const bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const { + LOCK(::cs_main); return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); } @@ -5214,6 +5246,7 @@ void ChainstateManager::Unload() void ChainstateManager::Reset() { + LOCK(::cs_main); m_ibd_chainstate.reset(); m_snapshot_chainstate.reset(); m_active_chainstate = nullptr; diff --git a/src/validation.h b/src/validation.h index 1b4e40cd53..e85c7bbf1a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -40,6 +40,7 @@ class CBlockIndex; class CBlockTreeDB; class CBlockUndo; class CChainParams; +struct CCheckpointData; class CInv; class CConnman; class CScriptCheck; @@ -143,14 +144,14 @@ extern const std::vector<std::string> CHECKLEVEL_DOC; FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos &pos); -/** Import blocks from an external file */ -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); -/** Run an instance of the script checking thread */ -void ThreadScriptCheck(int worker_num); +/** Run instances of script checking worker threads */ +void StartScriptCheckWorkerThreads(int threads_num); +/** Stop all of the script checking worker threads */ +void StopScriptCheckWorkerThreads(); /** * Return transaction from the block at block_index. * If block_index is not provided, fall back to mempool. @@ -164,13 +165,6 @@ void ThreadScriptCheck(int worker_num); * @returns The tx if found, otherwise nullptr */ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); -/** - * Find the best known block, and make it the tip of the block chain - * - * May not be called with cs_main held. May not be called in a - * validationinterface callback. - */ -bool ActivateBestChain(BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ @@ -289,7 +283,13 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Check a block is completely valid from start to finish (only works on top of our current best block) */ -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW = true, + bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ @@ -309,11 +309,6 @@ public: bool VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth); }; -CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** Find the last common block between the parameter chain and a locator. */ -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - enum DisconnectResult { DISCONNECT_OK, // All good. @@ -431,6 +426,21 @@ public: const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Find the last common block between the parameter chain and a locator. */ + CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Returns last CBlockIndex* that is a checkpoint + CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * Return the spend height, which is one more than the inputs.GetBestBlock(). + * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) + * This is also true for mempool checks. + */ + int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + ~BlockManager() { Unload(); } @@ -523,11 +533,6 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! Reference to a BlockManager instance which itself is shared across all - //! CChainState instances. Keeping a local reference allows us to test more - //! easily as opposed to referencing a global. - BlockManager& m_blockman; - //! mempool that is kept in sync with the chain CTxMemPool& m_mempool; @@ -535,6 +540,10 @@ protected: std::unique_ptr<CoinsViews> m_coins_views; public: + //! Reference to a BlockManager instance which itself is shared across all + //! CChainState instances. + BlockManager& m_blockman; + explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** @@ -611,6 +620,9 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Import blocks from an external file */ + void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -636,9 +648,10 @@ public: void PruneAndFlush(); /** - * Make the best chain active, in multiple steps. The result is either failure - * or an activated best chain. pblock is either nullptr or a pointer to a block - * that is already loaded (to avoid loading it again from disk). + * Find the best known block, and make it the tip of the block chain. The + * result is either failure or an activated best chain. pblock is either + * nullptr or a pointer to a block that is already loaded (to avoid loading + * it again from disk). * * ActivateBestChain is split into steps (see ActivateBestChainStep) so that * we avoid holding cs_main for an extended period of time; the length of this @@ -652,7 +665,7 @@ public: bool ActivateBestChain( BlockValidationState& state, const CChainParams& chainparams, - std::shared_ptr<const CBlock> pblock) LOCKS_EXCLUDED(cs_main); + std::shared_ptr<const CBlock> pblock = nullptr) LOCKS_EXCLUDED(cs_main); bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -789,7 +802,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_ibd_chainstate; + std::unique_ptr<CChainState> m_ibd_chainstate GUARDED_BY(::cs_main); //! A chainstate initialized on the basis of a UTXO snapshot. If this is //! non-null, it is always our active chainstate. @@ -802,7 +815,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_snapshot_chainstate; + std::unique_ptr<CChainState> m_snapshot_chainstate GUARDED_BY(::cs_main); //! Points to either the ibd or snapshot chainstate; indicates our //! most-work chain. @@ -815,7 +828,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - CChainState* m_active_chainstate{nullptr}; + CChainState* m_active_chainstate GUARDED_BY(::cs_main) {nullptr}; //! If true, the assumed-valid chainstate has been fully validated //! by the background validation chainstate. @@ -943,13 +956,6 @@ CChain& ChainActive(); /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; -/** - * Return the spend height, which is one more than the inputs.GetBestBlock(). - * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) - * This is also true for mempool checks. - */ -int GetSpendHeight(const CCoinsViewCache& inputs); - extern VersionBitsCache versionbitscache; /** diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 6ed48593fb..ad40e6da9a 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -488,9 +488,9 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) break; } if (pszSkip && - strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) + strncmp((const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) continue; - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { + if (strncmp((const char*)ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; @@ -723,6 +723,23 @@ bool BerkeleyBatch::TxnAbort() return (ret == 0); } +bool BerkeleyDatabaseSanityCheck() +{ + int major, minor; + DbEnv::version(&major, &minor, nullptr); + + /* If the major version differs, or the minor version of library is *older* + * than the header that was compiled against, flag an error. + */ + if (major != DB_VERSION_MAJOR || minor < DB_VERSION_MINOR) { + LogPrintf("BerkeleyDB database version conflict: header version is %d.%d, library version is %d.%d\n", + DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor); + return false; + } + + return true; +} + std::string BerkeleyDatabaseVersion() { return DbEnv::version(nullptr, nullptr, nullptr); diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index bf1617d67f..a8209587d7 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -223,6 +223,10 @@ public: std::string BerkeleyDatabaseVersion(); +/** Perform sanity check of runtime BDB version versus linked BDB version. + */ +bool BerkeleyDatabaseSanityCheck(); + //! Return object giving access to Berkeley database at specified path. std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index a2dea84d17..10f89e3a6f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -300,8 +300,26 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group ******************************************************************************/ -void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) { +void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only) { + // Compute the effective value first + const CAmount coin_fee = output.m_input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.m_input_bytes); + const CAmount ev = output.txout.nValue - coin_fee; + + // Filter for positive only here before adding the coin + if (positive_only && ev <= 0) return; + m_outputs.push_back(output); + CInputCoin& coin = m_outputs.back(); + + coin.m_fee = coin_fee; + fee += coin.m_fee; + + coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.m_input_bytes); + long_term_fee += coin.m_long_term_fee; + + coin.effective_value = ev; + effective_value += coin.effective_value; + m_from_me &= from_me; m_value += output.txout.nValue; m_depth = std::min(m_depth, depth); @@ -312,20 +330,6 @@ void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size // descendants is the count as seen from the top ancestor, not the descendants as seen from the // coin itself; thus, this value is counted as the max, not the sum m_descendants = std::max(m_descendants, descendants); - effective_value += output.effective_value; - fee += output.m_fee; - long_term_fee += output.m_long_term_fee; -} - -std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) { - auto it = m_outputs.begin(); - while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it; - if (it == m_outputs.end()) return it; - m_value -= output.txout.nValue; - effective_value -= output.effective_value; - fee -= output.m_fee; - long_term_fee -= output.m_long_term_fee; - return m_outputs.erase(it); } bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const @@ -334,35 +338,3 @@ bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_f && m_ancestors <= eligibility_filter.max_ancestors && m_descendants <= eligibility_filter.max_descendants; } - -void OutputGroup::SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate) -{ - fee = 0; - long_term_fee = 0; - effective_value = 0; - for (CInputCoin& coin : m_outputs) { - coin.m_fee = coin.m_input_bytes < 0 ? 0 : effective_feerate.GetFee(coin.m_input_bytes); - fee += coin.m_fee; - - coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); - long_term_fee += coin.m_long_term_fee; - - coin.effective_value = coin.txout.nValue - coin.m_fee; - effective_value += coin.effective_value; - } -} - -OutputGroup OutputGroup::GetPositiveOnlyGroup() -{ - OutputGroup group(*this); - for (auto it = group.m_outputs.begin(); it != group.m_outputs.end(); ) { - const CInputCoin& coin = *it; - // Only include outputs that are positive effective value (i.e. not dust) - if (coin.effective_value <= 0) { - it = group.Discard(coin); - } else { - ++it; - } - } - return group; -} diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index cf9768b927..f0e1addaf1 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -6,11 +6,10 @@ #define BITCOIN_WALLET_COINSELECTION_H #include <amount.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -class CFeeRate; - //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -63,9 +62,11 @@ struct CoinEligibilityFilter const int conf_theirs; const uint64_t max_ancestors; const uint64_t max_descendants; + const bool m_include_partial_groups{false}; //! Include partial destination groups when avoid_reuse and there are full groups CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} + CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants, bool include_partial) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants), m_include_partial_groups(include_partial) {} }; struct OutputGroup @@ -78,27 +79,18 @@ struct OutputGroup size_t m_descendants{0}; CAmount effective_value{0}; CAmount fee{0}; + CFeeRate m_effective_feerate{0}; CAmount long_term_fee{0}; + CFeeRate m_long_term_feerate{0}; OutputGroup() {} - OutputGroup(std::vector<CInputCoin>&& outputs, bool from_me, CAmount value, int depth, size_t ancestors, size_t descendants) - : m_outputs(std::move(outputs)) - , m_from_me(from_me) - , m_value(value) - , m_depth(depth) - , m_ancestors(ancestors) - , m_descendants(descendants) + OutputGroup(const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate) : + m_effective_feerate(effective_feerate), + m_long_term_feerate(long_term_feerate) {} - OutputGroup(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) : OutputGroup() { - Insert(output, depth, from_me, ancestors, descendants); - } - void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants); - std::vector<CInputCoin>::iterator Discard(const CInputCoin& output); - bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; - //! Update the OutputGroup's fee, long_term_fee, and effective_value based on the given feerates - void SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate); - OutputGroup GetPositiveOnlyGroup(); + void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only); + bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; }; bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees); diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 085dde1026..0d2be64dfb 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -86,6 +86,11 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const bool WalletInit::ParameterInteraction() const { +#ifdef USE_BDB + if (!BerkeleyDatabaseSanityCheck()) { + return InitError(Untranslated("A version conflict was detected between the run-time BerkeleyDB library and the one used during compilation.")); + } +#endif if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { for (const std::string& wallet : gArgs.GetArgs("-wallet")) { LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet); diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 5cdd7dff80..38ed7e7770 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -14,7 +14,27 @@ class CWallet; class CScript; -/** IsMine() return codes */ +/** + * IsMine() return codes, which depend on ScriptPubKeyMan implementation. + * Not every ScriptPubKeyMan covers all types, please refer to + * https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.21.0.md#ismine-semantics + * for better understanding. + * + * For LegacyScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_WATCH_ONLY: the scriptPubKey has been imported into the wallet; + * ISMINE_SPENDABLE: the scriptPubKey corresponds to an address owned by the wallet user (can spend with the private key); + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user; + * ISMINE_ALL: all ISMINE flags except for USED; + * ISMINE_ALL_USED: all ISMINE flags including USED; + * ISMINE_ENUM_ELEMENTS: the number of isminetype enum elements. + * + * For DescriptorScriptPubKeyMan and future ScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_SPENDABLE: the scriptPubKey matches a scriptPubKey in the wallet. + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user. + * + */ enum isminetype : unsigned int { ISMINE_NO = 0, diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 036fd4956f..30832f983b 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -62,7 +62,7 @@ bool VerifyWallets(interfaces::Chain& chain) std::set<fs::path> wallet_paths; for (const auto& wallet_file : gArgs.GetArgs("-wallet")) { - const fs::path path = fs::absolute(wallet_file, GetWalletDir()); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_file); if (!wallet_paths.insert(path).second) { chain.initWarning(strprintf(_("Ignoring duplicate -wallet %s."), wallet_file)); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6b46868d10..99803a91d2 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -934,9 +934,9 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TxoutType::NONSTANDARD: case TxoutType::WITNESS_UNKNOWN: case TxoutType::WITNESS_V1_TAPROOT: - default: return "unrecognized script"; - } + } // no default case, so the compiler can warn about missing cases + CHECK_NONFATAL(false); } static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) @@ -1740,3 +1740,72 @@ RPCHelpMan importdescriptors() }, }; } + +RPCHelpMan listdescriptors() +{ + return RPCHelpMan{ + "listdescriptors", + "\nList descriptors imported into a descriptor-enabled wallet.", + {}, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array of descriptor objects", + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "desc", "Descriptor string representation"}, + {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, + {RPCResult::Type::BOOL, "active", "Activeness flag"}, + {RPCResult::Type::BOOL, "internal", true, "Whether this is internal or external descriptor; defined only for active descriptors"}, + {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { + {RPCResult::Type::NUM, "", "Range start inclusive"}, + {RPCResult::Type::NUM, "", "Range end inclusive"}, + }}, + {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("listdescriptors", "") + HelpExampleRpc("listdescriptors", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + + if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "listdescriptors is not available for non-descriptor wallets"); + } + + LOCK(wallet->cs_wallet); + + UniValue response(UniValue::VARR); + const auto active_spk_mans = wallet->GetActiveScriptPubKeyMans(); + for (const auto& spk_man : wallet->GetAllScriptPubKeyMans()) { + const auto desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); + if (!desc_spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "Unexpected ScriptPubKey manager type."); + } + UniValue spk(UniValue::VOBJ); + LOCK(desc_spk_man->cs_desc_man); + const auto& wallet_descriptor = desc_spk_man->GetWalletDescriptor(); + spk.pushKV("desc", wallet_descriptor.descriptor->ToString()); + spk.pushKV("timestamp", wallet_descriptor.creation_time); + const bool active = active_spk_mans.count(desc_spk_man) != 0; + spk.pushKV("active", active); + const auto& type = wallet_descriptor.descriptor->GetOutputType(); + if (active && type != nullopt) { + spk.pushKV("internal", wallet->GetScriptPubKeyMan(*type, true) == desc_spk_man); + } + if (wallet_descriptor.descriptor->IsRange()) { + UniValue range(UniValue::VARR); + range.push_back(wallet_descriptor.range_start); + range.push_back(wallet_descriptor.range_end - 1); + spk.pushKV("range", range); + spk.pushKV("next", wallet_descriptor.next_index); + } + response.push_back(spk); + } + + return response; +}, + }; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 7705794c63..46de273d63 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2615,7 +2615,18 @@ static RPCHelpMan loadwallet() if (!wallet) { // Map bad format to not found, since bad format is returned when the // wallet directory exists, but doesn't contain a data file. - RPCErrorCode code = status == DatabaseStatus::FAILED_NOT_FOUND || status == DatabaseStatus::FAILED_BAD_FORMAT ? RPC_WALLET_NOT_FOUND : RPC_WALLET_ERROR; + RPCErrorCode code = RPC_WALLET_ERROR; + switch (status) { + case DatabaseStatus::FAILED_NOT_FOUND: + case DatabaseStatus::FAILED_BAD_FORMAT: + code = RPC_WALLET_NOT_FOUND; + break; + case DatabaseStatus::FAILED_ALREADY_LOADED: + code = RPC_WALLET_ALREADY_LOADED; + break; + default: // RPC_WALLET_ERROR is returned for all other cases. + break; + } throw JSONRPCError(code, error.original); } @@ -3449,10 +3460,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) CWallet* const pwallet = wallet.get(); if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { - if (!pwallet->chain().rpcEnableDeprecated("bumpfee")) { - throw JSONRPCError(RPC_METHOD_DEPRECATED, "Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22"); - } - want_psbt = true; + throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); @@ -3823,13 +3831,19 @@ RPCHelpMan getaddressinfo() LOCK(pwallet->cs_wallet); - UniValue ret(UniValue::VOBJ); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + // Make sure the destination is valid if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + // Set generic error message in case 'DecodeDestination' didn't set it + if (error_msg.empty()) error_msg = "Invalid address"; + + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error_msg); } + UniValue ret(UniValue::VOBJ); + std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -4537,73 +4551,75 @@ RPCHelpMan importprunedfunds(); RPCHelpMan removeprunedfunds(); RPCHelpMan importmulti(); RPCHelpMan importdescriptors(); +RPCHelpMan listdescriptors(); Span<const CRPCCommand> GetWalletRPCCommands() { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, - { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, - { "wallet", "abortrescan", &abortrescan, {} }, - { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, - { "wallet", "backupwallet", &backupwallet, {"destination"} }, - { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "psbtbumpfee", &psbtbumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors", "load_on_startup"} }, - { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, - { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, - { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, - { "wallet", "getaddressesbylabel", &getaddressesbylabel, {"label"} }, - { "wallet", "getaddressinfo", &getaddressinfo, {"address"} }, - { "wallet", "getbalance", &getbalance, {"dummy","minconf","include_watchonly","avoid_reuse"} }, - { "wallet", "getnewaddress", &getnewaddress, {"label","address_type"} }, - { "wallet", "getrawchangeaddress", &getrawchangeaddress, {"address_type"} }, - { "wallet", "getreceivedbyaddress", &getreceivedbyaddress, {"address","minconf"} }, - { "wallet", "getreceivedbylabel", &getreceivedbylabel, {"label","minconf"} }, - { "wallet", "gettransaction", &gettransaction, {"txid","include_watchonly","verbose"} }, - { "wallet", "getunconfirmedbalance", &getunconfirmedbalance, {} }, - { "wallet", "getbalances", &getbalances, {} }, - { "wallet", "getwalletinfo", &getwalletinfo, {} }, - { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, - { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, - { "wallet", "importmulti", &importmulti, {"requests","options"} }, - { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, - { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, - { "wallet", "importpubkey", &importpubkey, {"pubkey","label","rescan"} }, - { "wallet", "importwallet", &importwallet, {"filename"} }, - { "wallet", "keypoolrefill", &keypoolrefill, {"newsize"} }, - { "wallet", "listaddressgroupings", &listaddressgroupings, {} }, - { "wallet", "listlabels", &listlabels, {"purpose"} }, - { "wallet", "listlockunspent", &listlockunspent, {} }, - { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, {"minconf","include_empty","include_watchonly","address_filter"} }, - { "wallet", "listreceivedbylabel", &listreceivedbylabel, {"minconf","include_empty","include_watchonly"} }, - { "wallet", "listsinceblock", &listsinceblock, {"blockhash","target_confirmations","include_watchonly","include_removed"} }, - { "wallet", "listtransactions", &listtransactions, {"label|dummy","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, - { "wallet", "listwalletdir", &listwalletdir, {} }, - { "wallet", "listwallets", &listwallets, {} }, - { "wallet", "loadwallet", &loadwallet, {"filename", "load_on_startup"} }, - { "wallet", "lockunspent", &lockunspent, {"unlock","transactions"} }, - { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, - { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, - { "wallet", "send", &send, {"outputs","conf_target","estimate_mode","fee_rate","options"} }, - { "wallet", "sendmany", &sendmany, {"dummy","amounts","minconf","comment","subtractfeefrom","replaceable","conf_target","estimate_mode","fee_rate","verbose"} }, - { "wallet", "sendtoaddress", &sendtoaddress, {"address","amount","comment","comment_to","subtractfeefromamount","replaceable","conf_target","estimate_mode","avoid_reuse","fee_rate","verbose"} }, - { "wallet", "sethdseed", &sethdseed, {"newkeypool","seed"} }, - { "wallet", "setlabel", &setlabel, {"address","label"} }, - { "wallet", "settxfee", &settxfee, {"amount"} }, - { "wallet", "setwalletflag", &setwalletflag, {"flag","value"} }, - { "wallet", "signmessage", &signmessage, {"address","message"} }, - { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, - { "wallet", "unloadwallet", &unloadwallet, {"wallet_name", "load_on_startup"} }, - { "wallet", "upgradewallet", &upgradewallet, {"version"} }, - { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, - { "wallet", "walletlock", &walletlock, {} }, - { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, - { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, - { "wallet", "walletprocesspsbt", &walletprocesspsbt, {"psbt","sign","sighashtype","bip32derivs"} }, +{ // category actor (function) + // ------------------ ------------------------ + { "rawtransactions", &fundrawtransaction, }, + { "wallet", &abandontransaction, }, + { "wallet", &abortrescan, }, + { "wallet", &addmultisigaddress, }, + { "wallet", &backupwallet, }, + { "wallet", &bumpfee, }, + { "wallet", &psbtbumpfee, }, + { "wallet", &createwallet, }, + { "wallet", &dumpprivkey, }, + { "wallet", &dumpwallet, }, + { "wallet", &encryptwallet, }, + { "wallet", &getaddressesbylabel, }, + { "wallet", &getaddressinfo, }, + { "wallet", &getbalance, }, + { "wallet", &getnewaddress, }, + { "wallet", &getrawchangeaddress, }, + { "wallet", &getreceivedbyaddress, }, + { "wallet", &getreceivedbylabel, }, + { "wallet", &gettransaction, }, + { "wallet", &getunconfirmedbalance, }, + { "wallet", &getbalances, }, + { "wallet", &getwalletinfo, }, + { "wallet", &importaddress, }, + { "wallet", &importdescriptors, }, + { "wallet", &importmulti, }, + { "wallet", &importprivkey, }, + { "wallet", &importprunedfunds, }, + { "wallet", &importpubkey, }, + { "wallet", &importwallet, }, + { "wallet", &keypoolrefill, }, + { "wallet", &listaddressgroupings, }, + { "wallet", &listdescriptors, }, + { "wallet", &listlabels, }, + { "wallet", &listlockunspent, }, + { "wallet", &listreceivedbyaddress, }, + { "wallet", &listreceivedbylabel, }, + { "wallet", &listsinceblock, }, + { "wallet", &listtransactions, }, + { "wallet", &listunspent, }, + { "wallet", &listwalletdir, }, + { "wallet", &listwallets, }, + { "wallet", &loadwallet, }, + { "wallet", &lockunspent, }, + { "wallet", &removeprunedfunds, }, + { "wallet", &rescanblockchain, }, + { "wallet", &send, }, + { "wallet", &sendmany, }, + { "wallet", &sendtoaddress, }, + { "wallet", &sethdseed, }, + { "wallet", &setlabel, }, + { "wallet", &settxfee, }, + { "wallet", &setwalletflag, }, + { "wallet", &signmessage, }, + { "wallet", &signrawtransactionwithwallet, }, + { "wallet", &unloadwallet, }, + { "wallet", &upgradewallet, }, + { "wallet", &walletcreatefundedpsbt, }, + { "wallet", &walletlock, }, + { "wallet", &walletpassphrase, }, + { "wallet", &walletpassphrasechange, }, + { "wallet", &walletprocesspsbt, }, }; // clang-format on return MakeSpan(commands); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 15972fe7bb..55110f30d9 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -94,8 +94,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s TxoutType whichType = Solver(scriptPubKey, vSolutions); CKeyID keyID; - switch (whichType) - { + switch (whichType) { case TxoutType::NONSTANDARD: case TxoutType::NULL_DATA: case TxoutType::WITNESS_UNKNOWN: @@ -194,7 +193,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s } break; } - } + } // no default case, so the compiler can warn about missing cases if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { ret = std::max(ret, IsMineResult::WATCH_ONLY); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 202804c9ff..ffac78d752 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -35,7 +35,7 @@ static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); CoinEligibilityFilter filter_confirmed(1, 1, 0); CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0); +CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0, false); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -115,7 +115,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<CInputCoin>& coins { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin, 0, true, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin, 0, true, 0, 0, false); + } return static_groups; } @@ -123,7 +126,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0, false); + } return static_groups; } @@ -263,22 +269,22 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } // Make sure that effective value is working in SelectCoinsMinConf when BnB is used - CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0); + CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0, false); CoinSet setCoinsRet; CAmount nValueRet; bool bnb_used; empty_wallet(); add_coin(1); vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); // Test fees subtracted from output: empty_wallet(); add_coin(1 * CENT); vCoins.at(0).nInputBytes = 40; - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); // Make sure that can use BnB when there are preset inputs @@ -317,24 +323,24 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) empty_wallet(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); add_coin(1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can find a new 1 cent - BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); add_coin(2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can make 3 cents of new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); add_coin(5*CENT); // add a mature 5 cent coin, @@ -344,33 +350,33 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -384,30 +390,30 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -416,11 +422,11 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin( 2*COIN); add_coin( 3*COIN); add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -435,14 +441,14 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided add_coin(1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: @@ -450,7 +456,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) @@ -459,7 +465,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(50000 * COIN); - BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -472,7 +478,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 7 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -482,7 +488,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 8 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 @@ -493,12 +499,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); } @@ -512,7 +518,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); if (amt - 2000 < MIN_CHANGE) { // needs more than one input: @@ -538,8 +544,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -547,8 +553,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -570,8 +576,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -598,7 +604,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(1000 * COIN); add_coin(3 * COIN); - BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -633,13 +639,13 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CAmount target = rand.randrange(balance - 1000) + 1000; // Perform selection - CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0); - CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0); + CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0, false); + CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0, false); CoinSet out_set; CAmount out_value = 0; bool bnb_used = false; - BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_bnb, bnb_used) || - testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_knapsack, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_bnb, bnb_used) || + testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_knapsack, bnb_used)); BOOST_CHECK_GE(out_value, target); } } diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 27179839b7..b2eb8e4bca 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -30,8 +30,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - BOOST_CHECK(filename == test_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, test_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_directory) @@ -41,8 +41,8 @@ BOOST_AUTO_TEST_CASE(getwalletenv_directory) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(datadir, filename); - BOOST_CHECK(filename == expected_name); - BOOST_CHECK(env->Directory() == datadir); + BOOST_CHECK_EQUAL(filename, expected_name); + BOOST_CHECK_EQUAL(env->Directory(), datadir); } BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_multiple) diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index aa7f1c83d8..f035a70a20 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <fs.h> +#include <univalue.h> #include <util/check.h> #include <util/system.h> @@ -37,6 +38,9 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam InitWalletDirTestingSetup::~InitWalletDirTestingSetup() { + gArgs.LockSettings([&](util::Settings& settings) { + settings.forced_settings.erase("walletdir"); + }); fs::current_path(m_cwd); } diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index e70b56c529..45e1b8c4b8 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -19,7 +19,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); - BOOST_CHECK(walletdir == expected_path); + BOOST_CHECK_EQUAL(walletdir, expected_path); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 918946f9e7..db80745db0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2357,13 +2357,12 @@ const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int out return ptx->vout[n]; } -bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, +bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const { setCoinsRet.clear(); nValueRet = 0; - std::vector<OutputGroup> utxo_pool; if (coin_selection_params.use_bnb) { // Get long term estimate FeeCalculation feeCalc; @@ -2371,35 +2370,27 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil temp.m_confirm_target = 1008; CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); - // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + // Get the feerate for effective value. + // When subtracting the fee from the outputs, we want the effective feerate to be 0 + CFeeRate effective_feerate{0}; + if (!coin_selection_params.m_subtract_fee_outputs) { + effective_feerate = coin_selection_params.effective_fee; + } - // Filter by the min conf specs and add to utxo_pool and calculate effective value - for (OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, effective_feerate, long_term_feerate, eligibility_filter, true /* positive_only */); - if (coin_selection_params.m_subtract_fee_outputs) { - // Set the effective feerate to 0 as we don't want to use the effective value since the fees will be deducted from the output - group.SetFees(CFeeRate(0) /* effective_feerate */, long_term_feerate); - } else { - group.SetFees(coin_selection_params.effective_fee, long_term_feerate); - } + // Calculate cost of change + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); - OutputGroup pos_group = group.GetPositiveOnlyGroup(); - if (pos_group.effective_value > 0) utxo_pool.push_back(pos_group); - } // Calculate the fees for things that aren't inputs CAmount not_input_fees = coin_selection_params.effective_fee.GetFee(coin_selection_params.tx_noinputs_size); bnb_used = true; - return SelectCoinsBnB(utxo_pool, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); + return SelectCoinsBnB(groups, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); } else { - // Filter by the min conf specs and add to utxo_pool - for (const OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; - utxo_pool.push_back(group); - } + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, CFeeRate(0), CFeeRate(0), eligibility_filter, false /* positive_only */); + bnb_used = false; - return KnapsackSolver(nTargetValue, utxo_pool, setCoinsRet, nValueRet); + return KnapsackSolver(nTargetValue, groups, setCoinsRet, nValueRet); } } @@ -2482,16 +2473,14 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); - bool res = value_to_select <= 0 || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset util::insert(setCoinsRet, setPresetCoins); @@ -2782,6 +2771,7 @@ bool CWallet::CreateTransactionInternal( std::vector<COutput> vAvailableCoins; AvailableCoins(vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy + coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; // Create change script that will be used if we need change // TODO: pass in scriptChange instead of reservedest so @@ -3780,7 +3770,7 @@ std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, cons // 2. Path to an existing directory. // 3. Path to a symlink to a directory. // 4. For backwards compatibility, the name of a data file in -walletdir. - const fs::path& wallet_path = fs::absolute(name, GetWalletDir()); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), name); fs::file_type path_type = fs::symlink_status(wallet_path).type(); if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || @@ -4192,51 +4182,90 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { - std::vector<OutputGroup> groups; - std::map<CTxDestination, OutputGroup> gmap; - std::set<CTxDestination> full_groups; +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const +{ + std::vector<OutputGroup> groups_out; - for (const auto& output : outputs) { - if (output.fSpendable) { - CTxDestination dst; - CInputCoin input_coin = output.GetInputCoin(); + if (separate_coins) { + // Single coin means no grouping. Each COutput gets its own OutputGroup. + for (const COutput& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - auto it = gmap.find(dst); - if (it != gmap.end()) { - // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES - // number of entries, to protect against inadvertently creating - // a too-large transaction when using -avoidpartialspends to - // prevent breaking consensus or surprising users with a very - // high amount of fees. - if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(it->second); - it->second = OutputGroup{}; - full_groups.insert(dst); - } - it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } else { - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } - } else { - groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } + CInputCoin input_coin = output.GetInputCoin(); + + // Make an OutputGroup containing just this output + OutputGroup group{effective_feerate, long_term_feerate}; + group.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); + } + return groups_out; + } + + // We want to combine COutputs that have the same scriptPubKey into single OutputGroups + // except when there are more than OUTPUT_GROUP_MAX_ENTRIES COutputs grouped in an OutputGroup. + // To do this, we maintain a map where the key is the scriptPubKey and the value is a vector of OutputGroups. + // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput's CInputCoin is added + // to the last OutputGroup in the vector for the scriptPubKey. When the last OutputGroup has + // OUTPUT_GROUP_MAX_ENTRIES CInputCoins, a new OutputGroup is added to the end of the vector. + std::map<CScript, std::vector<OutputGroup>> spk_to_groups_map; + for (const auto& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; + + size_t ancestors, descendants; + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + CInputCoin input_coin = output.GetInputCoin(); + CScript spk = input_coin.txout.scriptPubKey; + + std::vector<OutputGroup>& groups = spk_to_groups_map[spk]; + + if (groups.size() == 0) { + // No OutputGroups for this scriptPubKey yet, add one + groups.emplace_back(effective_feerate, long_term_feerate); } + + // Get the last OutputGroup in the vector so that we can add the CInputCoin to it + // A pointer is used here so that group can be reassigned later if it is full. + OutputGroup* group = &groups.back(); + + // Check if this OutputGroup is full. We limit to OUTPUT_GROUP_MAX_ENTRIES when using -avoidpartialspends + // to avoid surprising users with very high fees. + if (group->m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + // The last output group is full, add a new group to the vector and use that group for the insertion + groups.emplace_back(effective_feerate, long_term_feerate); + group = &groups.back(); + } + + // Add the input_coin to group + group->Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); } - if (!single_coin) { - for (auto& it : gmap) { - auto& group = it.second; - if (full_groups.count(it.first) > 0) { - // Make this unattractive as we want coin selection to avoid it if possible - group.m_ancestors = max_ancestors - 1; + + // Now we go through the entire map and pull out the OutputGroups + for (const auto& spk_and_groups_pair: spk_to_groups_map) { + const std::vector<OutputGroup>& groups_per_spk= spk_and_groups_pair.second; + + // Go through the vector backwards. This allows for the first item we deal with being the partial group. + for (auto group_it = groups_per_spk.rbegin(); group_it != groups_per_spk.rend(); group_it++) { + const OutputGroup& group = *group_it; + + // Don't include partial groups if there are full groups too and we don't want partial groups + if (group_it == groups_per_spk.rbegin() && groups_per_spk.size() > 1 && !filter.m_include_partial_groups) { + continue; } - groups.push_back(group); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); } } - return groups; + + return groups_out; } bool CWallet::IsCrypted() const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e3eae1dd95..4fc4466604 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -610,8 +610,16 @@ struct CoinSelectionParams size_t tx_noinputs_size = 0; //! Indicate that we are subtracting the fee from outputs bool m_subtract_fee_outputs = false; - - CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size) : use_bnb(use_bnb), change_output_size(change_output_size), change_spend_size(change_spend_size), effective_fee(effective_fee), tx_noinputs_size(tx_noinputs_size) {} + bool m_avoid_partial_spends = false; + + CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size, bool avoid_partial) : + use_bnb(use_bnb), + change_output_size(change_output_size), + change_spend_size(change_spend_size), + effective_fee(effective_fee), + tx_noinputs_size(tx_noinputs_size), + m_avoid_partial_spends(avoid_partial) + {} CoinSelectionParams() {} }; @@ -818,7 +826,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, + bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -827,7 +835,7 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const; bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index a1bb7343f4..b2cb0bf479 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -103,10 +103,8 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& name) +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) { - fs::path path = fs::absolute(name, GetWalletDir()); - if (args.IsArgSet("-format") && command != "createfromdump") { tfm::format(std::cerr, "The -format option can only be used with the \"createfromdump\" command.\n"); return false; @@ -119,6 +117,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (command == "create" && !args.IsArgSet("-wallet")) { + tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + return false; + } + const std::string name = args.GetArg("-wallet", ""); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); if (command == "create") { DatabaseOptions options; diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index f544a6f727..f4516bb5bc 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -10,7 +10,7 @@ namespace WalletTool { void WalletShowInfo(CWallet* wallet_instance); -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& file); +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command); } // namespace WalletTool diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index c0207f9dd6..168ba841c8 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -17,6 +17,7 @@ #include <cstdarg> #include <cstddef> #include <map> +#include <optional> #include <string> #include <utility> @@ -227,50 +228,43 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } +// Helper function to send a 'sequence' topic message with the following structure: +// <32-byte hash> | <1-byte label> | <8-byte LE sequence> (optional) +static bool SendSequenceMsg(CZMQAbstractPublishNotifier& notifier, uint256 hash, char label, std::optional<uint64_t> sequence = {}) +{ + unsigned char data[sizeof(hash) + sizeof(label) + sizeof(uint64_t)]; + for (unsigned int i = 0; i < sizeof(hash); ++i) { + data[sizeof(hash) - 1 - i] = hash.begin()[i]; + } + data[sizeof(hash)] = label; + if (sequence) WriteLE64(data + sizeof(hash) + sizeof(label), *sequence); + return notifier.SendZmqMessage(MSG_SEQUENCE, data, sequence ? sizeof(data) : sizeof(hash) + sizeof(label)); +} -// TODO: Dedup this code to take label char, log string bool CZMQPublishSequenceNotifier::NotifyBlockConnect(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); LogPrint(BCLog::ZMQ, "zmq: Publish sequence block connect %s to %s\n", hash.GetHex(), this->address); - char data[sizeof(uint256)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(data) - 1] = 'C'; // Block (C)onnect - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Block (C)onnect */ 'C'); } bool CZMQPublishSequenceNotifier::NotifyBlockDisconnect(const CBlockIndex *pindex) { uint256 hash = pindex->GetBlockHash(); LogPrint(BCLog::ZMQ, "zmq: Publish sequence block disconnect %s to %s\n", hash.GetHex(), this->address); - char data[sizeof(uint256)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(data) - 1] = 'D'; // Block (D)isconnect - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Block (D)isconnect */ 'D'); } bool CZMQPublishSequenceNotifier::NotifyTransactionAcceptance(const CTransaction &transaction, uint64_t mempool_sequence) { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "zmq: Publish hashtx mempool acceptance %s to %s\n", hash.GetHex(), this->address); - unsigned char data[sizeof(uint256)+sizeof(mempool_sequence)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(uint256)] = 'A'; // Mempool (A)cceptance - WriteLE64(data+sizeof(uint256)+1, mempool_sequence); - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Mempool (A)cceptance */ 'A', mempool_sequence); } bool CZMQPublishSequenceNotifier::NotifyTransactionRemoval(const CTransaction &transaction, uint64_t mempool_sequence) { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "zmq: Publish hashtx mempool removal %s to %s\n", hash.GetHex(), this->address); - unsigned char data[sizeof(uint256)+sizeof(mempool_sequence)+1]; - for (unsigned int i = 0; i < sizeof(uint256); i++) - data[sizeof(uint256) - 1 - i] = hash.begin()[i]; - data[sizeof(uint256)] = 'R'; // Mempool (R)emoval - WriteLE64(data+sizeof(uint256)+1, mempool_sequence); - return SendZmqMessage(MSG_SEQUENCE, data, sizeof(data)); + return SendSequenceMsg(*this, hash, /* Mempool (R)emoval */ 'R', mempool_sequence); } diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 1dd751b493..81859f924f 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -52,9 +52,9 @@ static RPCHelpMan getzmqnotifications() } const CRPCCommand commands[] = -{ // category name actor (function) argNames - // ----------------- ------------------------ ----------------------- ---------- - { "zmq", "getzmqnotifications", &getzmqnotifications, {} }, +{ // category actor (function) + // ----------------- ----------------------- + { "zmq", &getzmqnotifications, }, }; } // anonymous namespace diff --git a/test/functional/README.md b/test/functional/README.md index 2d04413eb2..d830ba0334 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -63,10 +63,13 @@ don't have test cases for. - Avoid stop-starting the nodes multiple times during the test if possible. A stop-start takes several seconds, so doing it several times blows up the runtime of the test. -- Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether - or not to use the cached data directories. The cached data directories - contain a 200-block pre-mined blockchain and wallets for four nodes. Each node - has 25 mature blocks (25x50=1250 BTC) in its wallet. +- Set the `self.setup_clean_chain` variable in `set_test_params()` to `True` to + initialize an empty blockchain and start from the Genesis block, rather than + load a premined blockchain from cache with the default value of `False`. The + cached data directories contain a 200-block pre-mined blockchain with the + spendable mining rewards being split between four nodes. Each node has 25 + mature block subsidies (25x50=1250 BTC) in its wallet. Using them is much more + efficient than mining blocks in your test. - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 6e72db1d96..fab921ef19 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -57,7 +57,7 @@ class BadTxTemplate: __metaclass__ = abc.ABCMeta # The expected error code given by bitcoind upon submission of the tx. - reject_reason = "" # type: Optional[str] + reject_reason: Optional[str] = "" # Only specified if it differs from mempool acceptance error. block_reject_reason = "" diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 97f24e1b6e..a0eb213a78 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -76,6 +76,9 @@ class ExampleTest(BitcoinTestFramework): """Override test parameters for your individual test. This method must be overridden and num_nodes must be explicitly set.""" + # By default every test loads a pre-mined chain of 200 blocks from cache. + # Set setup_clean_chain to True to skip this and start from the Genesis + # block. self.setup_clean_chain = True self.num_nodes = 3 # Use self.extra_args to change command-line arguments for the nodes diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 2c6553fbe2..5fcecb4882 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -36,7 +36,6 @@ def expected_messages(filename): class AsmapTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def test_without_asmap_arg(self): diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 603d7f5d3b..1a148f04f4 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -29,9 +29,11 @@ Start three nodes: block 200. node2 will reject block 102 since it's assumed valid, but it isn't buried by at least two weeks' work. """ -import time -from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, @@ -79,24 +81,6 @@ class AssumeValidTest(BitcoinTestFramework): assert not p2p_conn.is_connected break - def assert_blockchain_height(self, node, height): - """Wait until the blockchain is no longer advancing and verify it's reached the expected height.""" - last_height = node.getblock(node.getbestblockhash())['height'] - timeout = 10 - while True: - time.sleep(0.25) - current_height = node.getblock(node.getbestblockhash())['height'] - if current_height != last_height: - last_height = current_height - if timeout < 0: - assert False, "blockchain too short after timeout: %d" % current_height - timeout - 0.25 - continue - elif current_height > height: - assert False, "blockchain too long: %d" % current_height - elif current_height == height: - break - def run_test(self): p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) @@ -177,7 +161,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node0. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p0) - self.assert_blockchain_height(self.nodes[0], 101) + self.wait_until(lambda: self.nodes[0].getblockcount() >= 101) + assert_equal(self.nodes[0].getblockcount(), 101) # Send all blocks to node1. All blocks will be accepted. for i in range(2202): @@ -188,7 +173,8 @@ class AssumeValidTest(BitcoinTestFramework): # Send blocks to node2. Block 102 will be rejected. self.send_blocks_until_disconnected(p2p2) - self.assert_blockchain_height(self.nodes[2], 101) + self.wait_until(lambda: self.nodes[2].getblockcount() >= 101) + assert_equal(self.nodes[2].getblockcount(), 101) if __name__ == '__main__': diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index b161c71a85..e6a53b52db 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -354,73 +354,75 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): hdkeypath = v17_info["hdkeypath"] pubkey = v17_info["pubkey"] - # Copy the 0.16 wallet to the last Bitcoin Core version and open it: - shutil.copyfile( - os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), - os.path.join(node_master_wallets_dir, "u1_v16") - ) - load_res = node_master.loadwallet("u1_v16") - # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 - assert_equal(load_res['warning'], '') - wallet = node_master.get_wallet_rpc("u1_v16") - info = wallet.getaddressinfo(v16_addr) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it - os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) - shutil.copyfile( - os.path.join(node_master_wallets_dir, "u1_v16"), - os.path.join(node_v16_wallets_dir, "wallets/u1_v16") - ) - self.start_node(-1, extra_args=["-wallet=u1_v16"]) - wallet = node_v16.get_wallet_rpc("u1_v16") - info = wallet.validateaddress(v16_addr) - assert_equal(info, v16_info) - - # Copy the 0.17 wallet to the last Bitcoin Core version and open it: - node_v17.unloadwallet("u1_v17") - shutil.copytree( - os.path.join(node_v17_wallets_dir, "u1_v17"), - os.path.join(node_master_wallets_dir, "u1_v17") - ) - node_master.loadwallet("u1_v17") - wallet = node_master.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" - assert_equal(info["desc"], descsum_create(descriptor)) - - # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it - node_master.unloadwallet("u1_v17") - shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "u1_v17"), - os.path.join(node_v17_wallets_dir, "u1_v17") - ) - node_v17.loadwallet("u1_v17") - wallet = node_v17.get_wallet_rpc("u1_v17") - info = wallet.getaddressinfo(address) - assert_equal(info, v17_info) - - # Copy the 0.19 wallet to the last Bitcoin Core version and open it: - shutil.copytree( - os.path.join(node_v19_wallets_dir, "w1_v19"), - os.path.join(node_master_wallets_dir, "w1_v19") - ) - node_master.loadwallet("w1_v19") - wallet = node_master.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + if self.is_bdb_compiled(): + # Old wallets are BDB and will only work if BDB is compiled + # Copy the 0.16 wallet to the last Bitcoin Core version and open it: + shutil.copyfile( + os.path.join(node_v16_wallets_dir, "wallets/u1_v16"), + os.path.join(node_master_wallets_dir, "u1_v16") + ) + load_res = node_master.loadwallet("u1_v16") + # Make sure this wallet opens without warnings. See https://github.com/bitcoin/bitcoin/pull/19054 + assert_equal(load_res['warning'], '') + wallet = node_master.get_wallet_rpc("u1_v16") + info = wallet.getaddressinfo(v16_addr) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it + os.remove(os.path.join(node_v16_wallets_dir, "wallets/u1_v16")) + shutil.copyfile( + os.path.join(node_master_wallets_dir, "u1_v16"), + os.path.join(node_v16_wallets_dir, "wallets/u1_v16") + ) + self.start_node(-1, extra_args=["-wallet=u1_v16"]) + wallet = node_v16.get_wallet_rpc("u1_v16") + info = wallet.validateaddress(v16_addr) + assert_equal(info, v16_info) - # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it - node_master.unloadwallet("w1_v19") - shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) - shutil.copytree( - os.path.join(node_master_wallets_dir, "w1_v19"), - os.path.join(node_v19_wallets_dir, "w1_v19") - ) - node_v19.loadwallet("w1_v19") - wallet = node_v19.get_wallet_rpc("w1_v19") - assert wallet.getaddressinfo(address_18075)["solvable"] + # Copy the 0.17 wallet to the last Bitcoin Core version and open it: + node_v17.unloadwallet("u1_v17") + shutil.copytree( + os.path.join(node_v17_wallets_dir, "u1_v17"), + os.path.join(node_master_wallets_dir, "u1_v17") + ) + node_master.loadwallet("u1_v17") + wallet = node_master.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" + assert_equal(info["desc"], descsum_create(descriptor)) + + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: + shutil.copytree( + os.path.join(node_v19_wallets_dir, "w1_v19"), + os.path.join(node_master_wallets_dir, "w1_v19") + ) + node_master.loadwallet("w1_v19") + wallet = node_master.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index aad255c4a9..b7c2887ee8 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -126,8 +126,13 @@ class BIP65Test(BitcoinTestFramework): # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) + [{ + 'txid': spendtx.hash, + 'wtxid': spendtx.getwtxid(), + 'allowed': False, + 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)', + }], + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 3e28dae4b3..2445b6d977 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -7,6 +7,7 @@ import os from test_framework.test_framework import BitcoinTestFramework +from test_framework import util class ConfArgsTest(BitcoinTestFramework): @@ -42,10 +43,11 @@ class ConfArgsTest(BitcoinTestFramework): conf.write("wallet=foo\n") self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on %s network when in [%s] section.' % (self.chain, self.chain)) + main_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_main.conf') + util.write_config(main_conf_file_path, n=0, chain='', extra_config='includeconf={}\n'.format(inc_conf_file_path)) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('regtest=0\n') # mainnet conf.write('acceptnonstdtxn=1\n') - self.nodes[0].assert_start_raises_init_error(expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') + self.nodes[0].assert_start_raises_init_error(extra_args=["-conf={}".format(main_conf_file_path)], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index f9ece244fb..2b56bc78f5 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -49,7 +49,6 @@ from test_framework.util import ( class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 - self.setup_clean_chain = False self.rpc_timeout = 480 self.supports_cli = False diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 3f7efdbded..3b430139b1 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -112,8 +112,13 @@ class BIP66Test(BitcoinTestFramework): # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) + [{ + 'txid': spendtx.hash, + 'wtxid': spendtx.getwtxid(), + 'allowed': False, + 'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)', + }], + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 7de9a589be..2798d11b0a 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -4,6 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" import os +import random +import string from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch @@ -27,11 +29,21 @@ class FilelockTest(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): - self.nodes[0].createwallet(self.default_wallet_name) - wallet_dir = os.path.join(datadir, 'wallets') - self.log.info("Check that we can't start a second bitcoind instance using the same wallet") - expected_msg = "Error: Error initializing wallet database environment" - self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + self.default_wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + def check_wallet_filelock(descriptors): + wallet_name = ''.join([random.choice(string.ascii_lowercase) for _ in range(6)]) + self.nodes[0].createwallet(wallet_name=wallet_name, descriptors=descriptors) + wallet_dir = os.path.join(datadir, 'wallets') + self.log.info("Check that we can't start a second bitcoind instance using the same wallet") + if descriptors: + expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + else: + expected_msg = "Error: Error initializing wallet database environment" + self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + + if self.is_bdb_compiled(): + check_wallet_filelock(False) + if self.is_sqlite_compiled(): + check_wallet_filelock(True) if __name__ == '__main__': FilelockTest().main() diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 6f1a0cd348..f22b7f266a 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -20,7 +20,6 @@ from test_framework.test_framework import BitcoinTestFramework class IncludeConfTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def setup_chain(self): diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index f2313bac13..b068ce612c 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -5,11 +5,11 @@ """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - hex_str_to_bytes, ) # Linux allow all characters other than \x00 @@ -49,6 +49,31 @@ class NotificationsTest(BitcoinTestFramework): super().setup_network() def run_test(self): + if self.is_wallet_compiled(): + # Setup the descriptors to be imported to the wallet + seed = "cTdGmKFWpbvpKQ7ejrdzqYT2hhjyb3GPHnLAK7wdi5Em67YLwSm9" + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + desc_imports = [{ + "desc": descsum_create("wpkh(" + xpriv + "/0/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + },{ + "desc": descsum_create("wpkh(" + xpriv + "/1/*)"), + "timestamp": 0, + "active": True, + "keypool": True, + "internal": True, + }] + # Make the wallets and import the descriptors + # Ensures that node 0 and node 1 share the same wallet for the conflicting transaction tests below. + for i, name in enumerate(self.wallet_names): + self.nodes[i].createwallet(wallet_name=name, descriptors=self.options.descriptors, blank=True, load_on_startup=True) + if self.options.descriptors: + self.nodes[i].importdescriptors(desc_imports) + else: + self.nodes[i].sethdseed(True, seed) + self.log.info("test -blocknotify") block_count = 10 blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) @@ -84,11 +109,10 @@ class NotificationsTest(BitcoinTestFramework): for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) - # Conflicting transactions tests. Give node 0 same wallet seed as - # node 1, generate spends from node 0, and check notifications + # Conflicting transactions tests. + # Generate spends from node 0, and check notifications # triggered by node 1 self.log.info("test -walletnotify with conflicting transactions") - self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) self.nodes[0].rescanblockchain() self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) self.sync_blocks() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 6ee2b72c11..5027a9828f 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -517,7 +517,6 @@ def add_spender(spenders, *args, **kwargs): def random_checksig_style(pubkey): """Creates a random CHECKSIG* tapscript that would succeed with only the valid signature on witness stack.""" - return bytes(CScript([pubkey, OP_CHECKSIG])) opcode = random.choice([OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKSIGADD]) if (opcode == OP_CHECKSIGVERIFY): ret = CScript([pubkey, opcode, OP_1]) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 1257dff1ae..2cf0ef2251 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -29,6 +29,8 @@ class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + if self.is_wallet_compiled(): + self.requires_wallet = True def skip_test_if_missing_module(self): self.skip_if_no_cli() diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 9b2414cf2d..e9f61be4d4 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -62,6 +62,8 @@ class ZMQSubscriber: class ZMQTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + if self.is_wallet_compiled(): + self.requires_wallet = True def skip_test_if_missing_module(self): self.skip_if_no_py3_zmq() @@ -80,34 +82,43 @@ class ZMQTest (BitcoinTestFramework): self.log.debug("Destroying ZMQ context") self.ctx.destroy(linger=None) + # Restart node with the specified zmq notifications enabled, subscribe to + # all of them and return the corresponding ZMQSubscriber objects. + def setup_zmq_test(self, services, recv_timeout=60, connect_nodes=False): + subscribers = [] + for topic, address in services: + socket = self.ctx.socket(zmq.SUB) + socket.set(zmq.RCVTIMEO, recv_timeout*1000) + subscribers.append(ZMQSubscriber(socket, topic.encode())) + + self.restart_node(0, ["-zmqpub%s=%s" % (topic, address) for topic, address in services]) + + if connect_nodes: + self.connect_nodes(0, 1) + + for i, sub in enumerate(subscribers): + sub.socket.connect(services[i][1]) + + # Relax so that the subscribers are ready before publishing zmq messages + sleep(0.2) + + return subscribers + def test_basic(self): # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = 'tcp://127.0.0.1:28332' - sockets = [] - subs = [] - services = [b"hashblock", b"hashtx", b"rawblock", b"rawtx"] - for service in services: - sockets.append(self.ctx.socket(zmq.SUB)) - sockets[-1].set(zmq.RCVTIMEO, 60000) - subs.append(ZMQSubscriber(sockets[-1], service)) - - # Subscribe to all available topics. + subs = self.setup_zmq_test( + [(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]], + connect_nodes=True) + hashblock = subs[0] hashtx = subs[1] rawblock = subs[2] rawtx = subs[3] - self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx, rawblock, rawtx]]) - self.connect_nodes(0, 1) - for socket in sockets: - socket.connect(address) - - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) - num_blocks = 5 self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) genhashes = self.nodes[0].generatetoaddress(num_blocks, ADDRESS_BCRT1_UNSPENDABLE) @@ -174,25 +185,10 @@ class ZMQTest (BitcoinTestFramework): address = 'tcp://127.0.0.1:28333' - services = [b"hashblock", b"hashtx"] - sockets = [] - subs = [] - for service in services: - sockets.append(self.ctx.socket(zmq.SUB)) - # 2 second timeout to check end of notifications - sockets[-1].set(zmq.RCVTIMEO, 2000) - subs.append(ZMQSubscriber(sockets[-1], service)) - - # Subscribe to all available topics. - hashblock = subs[0] - hashtx = subs[1] - # Should only notify the tip if a reorg occurs - self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx]]) - for socket in sockets: - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + hashblock, hashtx = self.setup_zmq_test( + [(topic, address) for topic in ["hashblock", "hashtx"]], + recv_timeout=2) # 2 second timeout to check end of notifications # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) @@ -240,15 +236,7 @@ class ZMQTest (BitcoinTestFramework): <32-byte hash>A<8-byte LE uint> : Transactionhash added mempool """ self.log.info("Testing 'sequence' publisher") - address = 'tcp://127.0.0.1:28333' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - seq = ZMQSubscriber(socket, b'sequence') - - self.restart_node(0, ['-zmqpub%s=%s' % (seq.topic.decode(), address)]) - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")]) # Mempool sequence number starts at 1 seq_num = 1 @@ -399,16 +387,7 @@ class ZMQTest (BitcoinTestFramework): return self.log.info("Testing 'mempool sync' usage of sequence notifier") - address = 'tcp://127.0.0.1:28333' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - seq = ZMQSubscriber(socket, b'sequence') - - self.restart_node(0, ['-zmqpub%s=%s' % (seq.topic.decode(), address)]) - self.connect_nodes(0, 1) - socket.connect(address) - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + [seq] = self.setup_zmq_test([("sequence", "tcp://127.0.0.1:28333")], connect_nodes=True) # In-memory counter, should always start at 1 next_mempool_seq = self.nodes[0].getrawmempool(mempool_sequence=True)["mempool_sequence"] @@ -508,26 +487,17 @@ class ZMQTest (BitcoinTestFramework): def test_multiple_interfaces(self): # Set up two subscribers with different addresses - subscribers = [] - for i in range(2): - address = 'tcp://127.0.0.1:%d' % (28334 + i) - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - hashblock = ZMQSubscriber(socket, b"hashblock") - socket.connect(address) - subscribers.append({'address': address, 'hashblock': hashblock}) - - self.restart_node(0, ['-zmqpub%s=%s' % (subscriber['hashblock'].topic.decode(), subscriber['address']) for subscriber in subscribers]) - - # Relax so that the subscriber is ready before publishing zmq messages - sleep(0.2) + subscribers = self.setup_zmq_test([ + ("hashblock", "tcp://127.0.0.1:28334"), + ("hashblock", "tcp://127.0.0.1:28335"), + ]) # Generate 1 block in nodes[0] and receive all notifications self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) # Should receive the same block hash on both subscribers - assert_equal(self.nodes[0].getbestblockhash(), subscribers[0]['hashblock'].receive().hex()) - assert_equal(self.nodes[0].getbestblockhash(), subscribers[1]['hashblock'].receive().hex()) + assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) + assert_equal(self.nodes[0].getbestblockhash(), subscribers[1].receive().hex()) if __name__ == '__main__': ZMQTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index fc4c775668..c4002f524a 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -51,6 +51,8 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def check_mempool_result(self, result_expected, *args, **kwargs): """Wrapper to check result of testmempoolaccept on node_0's mempool""" result_test = self.nodes[0].testmempoolaccept(*args, **kwargs) + for r in result_test: + r.pop('wtxid') # Skip check for now assert_equal(result_expected, result_test) assert_equal(self.nodes[0].getmempoolinfo()['size'], self.mempool_size) # Must not change mempool state diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 70cd4ebb3b..752b925b92 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -69,6 +69,8 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + total_fee_old = self.nodes[0].getmempoolinfo()['total_fee'] + self.log.debug("Prioritize a transaction on node0") fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'], fees['modified']) @@ -76,6 +78,10 @@ class MempoolPersistTest(BitcoinTestFramework): fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.info('Check the total base fee is unchanged after prioritisetransaction') + assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee']) + assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items())) + tx_creation_time = self.nodes[0].getmempoolentry(txid=last_txid)['time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py index a63c3a3287..a04ba5db2d 100755 --- a/test/functional/p2p_add_connections.py +++ b/test/functional/p2p_add_connections.py @@ -17,7 +17,6 @@ def check_node_connections(*, node, num_in, num_out): class P2PAddConnections(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def setup_network(self): diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 91fbd722cf..69821763bd 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -46,7 +46,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index c592ab52b1..6584efae79 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -15,7 +15,6 @@ from test_framework.util import assert_equal class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [["-blocksonly"]] diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 642a217047..458e5235b6 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -81,7 +81,6 @@ class P2PBloomFilter(P2PInterface): class FilterTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [[ '-peerbloomfilters', diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index 2b75ad5175..d375af6fe1 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -41,7 +41,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index e4fc9fd178..f884cf90ff 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -13,7 +13,6 @@ from test_framework.test_framework import BitcoinTestFramework class InvalidLocatorTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = False def run_test(self): node = self.nodes[0] # convenience reference to the node diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py new file mode 100755 index 0000000000..080b2d93ad --- /dev/null +++ b/test/functional/p2p_message_capture.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test per-peer message capture capability. + +Additionally, the output of contrib/message-capture/message-capture-parser.py should be verified manually. +""" + +import glob +from io import BytesIO +import os + +from test_framework.p2p import P2PDataStore, MESSAGEMAP +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +def mini_parser(dat_file): + """Parse a data file created by CaptureMessage. + + From the data file we'll only check the structure. + + We won't care about things like: + - Deserializing the payload of the message + - This is managed by the deserialize methods in test_framework.messages + - The order of the messages + - There's no reason why we can't, say, change the order of the messages in the handshake + - Message Type + - We can add new message types + + We're ignoring these because they're simply too brittle to test here. + """ + with open(dat_file, 'rb') as f_in: + # This should have at least one message in it + assert(os.fstat(f_in.fileno()).st_size >= TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + while True: + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + tmp_header.read(TIME_SIZE) # skip the timestamp field + raw_msgtype = tmp_header.read(MSGTYPE_SIZE) + msgtype: bytes = raw_msgtype.split(b'\x00', 1)[0] + remainder = raw_msgtype.split(b'\x00', 1)[1] + assert(len(msgtype) > 0) + assert(msgtype in MESSAGEMAP) + assert(len(remainder) == 0 or not remainder.decode().isprintable()) + length: int = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") + data = f_in.read(length) + assert_equal(len(data), length) + + + +class MessageCaptureTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-capturemessages"]] + self.setup_clean_chain = True + + def run_test(self): + capturedir = os.path.join(self.nodes[0].datadir, "regtest/message_capture") + # Connect a node so that the handshake occurs + self.nodes[0].add_p2p_connection(P2PDataStore()) + self.nodes[0].disconnect_p2ps() + recv_file = glob.glob(os.path.join(capturedir, "*/msgs_recv.dat"))[0] + mini_parser(recv_file) + sent_file = glob.glob(os.path.join(capturedir, "*/msgs_sent.dat"))[0] + mini_parser(sent_file) + + +if __name__ == '__main__': + MessageCaptureTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index a9d8b12d70..54891b07e1 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -686,13 +686,35 @@ class SegWitTest(BitcoinTestFramework): if not self.segwit_active: # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00001000')}}]) + assert_equal( + self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + [{ + 'txid': tx3.hash, + 'wtxid': tx3.getwtxid(), + 'allowed': True, + 'vsize': tx3.get_vsize(), + 'fees': { + 'base': Decimal('0.00001000'), + }, + }], + ) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00011000')}}]) + assert_equal( + self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), + [{ + 'txid': tx3.hash, + 'wtxid': tx3.getwtxid(), + 'allowed': True, + 'vsize': tx3.get_vsize(), + 'fees': { + 'base': Decimal('0.00011000'), + }, + }], + ) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 47832b04bf..a7e240dcfa 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -74,9 +74,9 @@ class TimeoutsTest(BitcoinTestFramework): no_version_node.send_message(msg_ping()) expected_timeout_logs = [ - "version handshake timeout from 0", - "socket no message in first 3 seconds, 1 0 from 1", - "socket no message in first 3 seconds, 0 0 from 2", + "version handshake timeout peer=0", + "socket no message in first 3 seconds, 1 0 peer=1", + "socket no message in first 3 seconds, 0 0 peer=2", ] with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 8a751c6b54..4bf96cb0e6 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -56,7 +56,6 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def test_tx_requests(self): diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 209d1182d7..1af79b9f7c 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error, find_vout_for_address class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): @@ -24,37 +23,7 @@ class DeprecatedRpcTest(BitcoinTestFramework): # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) # self.nodes[1].generate(1) - if self.is_wallet_compiled(): - self.log.info("Test bumpfee RPC") - self.nodes[0].generate(101) - self.nodes[0].createwallet(wallet_name='nopriv', disable_private_keys=True) - noprivs0 = self.nodes[0].get_wallet_rpc('nopriv') - w0 = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - self.nodes[1].createwallet(wallet_name='nopriv', disable_private_keys=True) - noprivs1 = self.nodes[1].get_wallet_rpc('nopriv') - - address = w0.getnewaddress() - desc = w0.getaddressinfo(address)['desc'] - change_addr = w0.getrawchangeaddress() - change_desc = w0.getaddressinfo(change_addr)['desc'] - txid = w0.sendtoaddress(address=address, amount=10) - vout = find_vout_for_address(w0, txid, address) - self.nodes[0].generate(1) - rawtx = w0.createrawtransaction([{'txid': txid, 'vout': vout}], {w0.getnewaddress(): 5}, 0, True) - rawtx = w0.fundrawtransaction(rawtx, {'changeAddress': change_addr}) - signed_tx = w0.signrawtransactionwithwallet(rawtx['hex'])['hex'] - - noprivs0.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) - noprivs1.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) - - txid = w0.sendrawtransaction(signed_tx) - self.sync_all() - - assert_raises_rpc_error(-32, 'Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22', noprivs0.bumpfee, txid) - bumped_psbt = noprivs1.bumpfee(txid) - assert 'psbt' in bumped_psbt - else: - self.log.info("No tested deprecated RPC methods") + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 81862ac69e..51b7efb4c3 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -14,7 +14,6 @@ from test_framework.util import assert_raises_rpc_error class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index c83157a843..1eefd109f8 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -7,7 +7,39 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +from collections import defaultdict import os +import re + + +def parse_string(s): + assert s[0] == '"' + assert s[-1] == '"' + return s[1:-1] + + +def process_mapping(fname): + """Find and parse conversion table in implementation file `fname`.""" + cmds = [] + in_rpcs = False + with open(fname, "r", encoding="utf8") as f: + for line in f: + line = line.rstrip() + if not in_rpcs: + if line == 'static const CRPCConvertParam vRPCConvertParams[] =': + in_rpcs = True + else: + if line.startswith('};'): + in_rpcs = False + elif '{' in line and '"' in line: + m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) + assert m, 'No match to table expression: %s' % line + name = parse_string(m.group(1)) + idx = int(m.group(2)) + argname = parse_string(m.group(3)) + cmds.append((name, idx, argname)) + assert not in_rpcs and cmds + return cmds class HelpRpcTest(BitcoinTestFramework): @@ -16,11 +48,43 @@ class HelpRpcTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + self.test_client_conversion_table() self.test_categories() self.dump_help() if self.is_wallet_compiled(): self.wallet_help() + def test_client_conversion_table(self): + file_conversion_table = os.path.join(self.config["environment"]["SRCDIR"], 'src', 'rpc', 'client.cpp') + mapping_client = process_mapping(file_conversion_table) + # Ignore echojson in client table + mapping_client = [m for m in mapping_client if m[0] != 'echojson'] + + mapping_server = self.nodes[0].help("dump_all_command_conversions") + # Filter all RPCs whether they need conversion + mapping_server_conversion = [tuple(m[:3]) for m in mapping_server if not m[3]] + + # Only check if all RPC methods have been compiled (i.e. wallet is enabled) + if self.is_wallet_compiled() and sorted(mapping_client) != sorted(mapping_server_conversion): + raise AssertionError("RPC client conversion table ({}) and RPC server named arguments mismatch!\n{}".format( + file_conversion_table, + set(mapping_client).symmetric_difference(mapping_server_conversion), + )) + + # Check for conversion difference by argument name. + # It is preferable for API consistency that arguments with the same name + # have the same conversion, so bin by argument name. + all_methods_by_argname = defaultdict(list) + converts_by_argname = defaultdict(list) + for m in mapping_server: + all_methods_by_argname[m[2]].append(m[0]) + converts_by_argname[m[2]].append(m[3]) + + for argname, convert in converts_by_argname.items(): + if all(convert) != any(convert): + # Only allow dummy to fail consistency check + assert argname == 'dummy', ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) + def test_categories(self): node = self.nodes[0] diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py new file mode 100755 index 0000000000..469d6bdb05 --- /dev/null +++ b/test/functional/rpc_invalid_address_message.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test error messages for 'getaddressinfo' and 'validateaddress' RPC commands.""" + +from test_framework.test_framework import BitcoinTestFramework + +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +BECH32_VALID = 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv' +BECH32_INVALID_SIZE = 'bcrt1sqqpl9r5c' +BECH32_INVALID_PREFIX = 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4' + +BASE58_VALID = 'mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJRfn' +BASE58_INVALID_PREFIX = '17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem' + +INVALID_ADDRESS = 'asfah14i8fajz0123f' + +class InvalidAddressErrorMessageTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_validateaddress(self): + node = self.nodes[0] + + # Bech32 + info = node.validateaddress(BECH32_INVALID_SIZE) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid Bech32 address data size') + + info = node.validateaddress(BECH32_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Bech32 address') + + info = node.validateaddress(BECH32_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Base58 + info = node.validateaddress(BASE58_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Base58-encoded address') + + info = node.validateaddress(BASE58_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Invalid address format + info = node.validateaddress(INVALID_ADDRESS) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid address format') + + def test_getaddressinfo(self): + node = self.nodes[0] + + assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) + + assert_raises_rpc_error(-5, "Invalid prefix for Bech32 address", node.getaddressinfo, BECH32_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid address format", node.getaddressinfo, INVALID_ADDRESS) + + def run_test(self): + self.test_validateaddress() + self.test_getaddressinfo() + + +if __name__ == '__main__': + InvalidAddressErrorMessageTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index de0b7f303f..cf46616681 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -25,6 +25,7 @@ from test_framework.util import ( assert_raises_rpc_error, p2p_port, ) +from test_framework.wallet import MiniWallet def assert_net_servicesnames(servicesflag, servicenames): @@ -48,6 +49,9 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + # We need miniwallet to make a transaction + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.generate(1) # Get out of IBD for the minfeefilter and getpeerinfo tests. self.nodes[0].generate(101) @@ -74,8 +78,7 @@ class NetTest(BitcoinTestFramework): def test_getpeerinfo(self): self.log.info("Test getpeerinfo") # Create a few getpeerinfo last_block/last_transaction values. - if self.is_wallet_compiled(): - self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results self.nodes[1].generate(1) self.sync_all() time_now = int(time.time()) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b364077a9a..ed6abaed78 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -24,7 +24,6 @@ MAX_BIP125_RBF_SEQUENCE = 0xfffffffd class PSBTTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 3 self.extra_args = [ ["-walletrbf=1"], diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index e86f91b1d0..6177970872 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -10,6 +10,7 @@ Test corresponds to code in rpc/server.cpp. import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_raises_rpc_error class UptimeTest(BitcoinTestFramework): @@ -18,8 +19,12 @@ class UptimeTest(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): + self._test_negative_time() self._test_uptime() + def _test_negative_time(self): + assert_raises_rpc_error(-8, "Mocktime can not be negative: -1.", self.nodes[0].setmocktime, -1) + def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index f6ea9ef682..b8e899d675 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -178,7 +178,7 @@ can be called after the TestShell is shut down. | `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | | `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | | `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | -| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `setup_clean_chain` | `False` | A 200-block-long chain is initialized from cache by default. Instead, `setup_clean_chain` initializes an empty blockchain if set to `True`. | | `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | | `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | | `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py index 9de358aa0a..97b9c1d6d0 100644 --- a/test/functional/test_framework/bdb.py +++ b/test/functional/test_framework/bdb.py @@ -51,7 +51,6 @@ def dump_leaf_page(data): page_info['pgno'] = pgno page_info['prev_pgno'] = prev_pgno page_info['next_pgno'] = next_pgno - page_info['entries'] = entries page_info['hf_offset'] = hf_offset page_info['level'] = level page_info['pg_type'] = pg_type diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index bab4ad0008..27a09ef86c 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -564,6 +564,9 @@ class CTransaction: def serialize(self): return self.serialize_with_witness() + def getwtxid(self): + return hash256(self.serialize())[::-1].hex() + # Recalculate the txid (transaction hash without witness) def rehash(self): self.sha256 = None @@ -579,7 +582,7 @@ class CTransaction: if self.sha256 is None: self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) - self.hash = encode(hash256(self.serialize_without_witness())[::-1], 'hex_codec').decode('ascii') + self.hash = hash256(self.serialize_without_witness())[::-1].hex() def is_valid(self): self.calc_sha256() @@ -1270,7 +1273,7 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire # note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("msgtype", "data") + __slots__ = ("data") def __init__(self, msgtype, data=None): self.msgtype = msgtype diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index be0e9f24e2..c35533698c 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -29,8 +29,6 @@ MAX_SCRIPT_ELEMENT_SIZE = 520 LOCKTIME_THRESHOLD = 500000000 ANNEX_TAG = 0x50 -OPCODE_NAMES = {} # type: Dict[CScriptOp, str] - LEAF_VERSION_TAPSCRIPT = 0xc0 def hash160(s): @@ -47,7 +45,6 @@ def bn2vch(v): # Serialize to bytes return encoded_v.to_bytes(n_bytes, 'little') -_opcode_instances = [] # type: List[CScriptOp] class CScriptOp(int): """A single script opcode""" __slots__ = () @@ -111,6 +108,9 @@ class CScriptOp(int): _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] +OPCODE_NAMES: Dict[CScriptOp, str] = {} +_opcode_instances: List[CScriptOp] = [] + # Populate opcode instance table for n in range(0xff + 1): CScriptOp(n) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 7c5317480c..70a9798449 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -18,6 +18,7 @@ import sys import tempfile import time +from typing import List from .authproxy import JSONRPCException from . import coverage from .p2p import NetworkThread @@ -89,14 +90,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" - chain = None # type: str - setup_clean_chain = None # type: bool - def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" - self.chain = 'regtest' - self.setup_clean_chain = False - self.nodes = [] + self.chain: str = 'regtest' + self.setup_clean_chain: bool = False + self.nodes: List[TestNode] = [] self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = True @@ -110,6 +108,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # skipped. If list is truncated, wallet creation is skipped and keys # are not imported. self.wallet_names = None + # By default the wallet is not required. Set to true by skip_if_no_wallet(). + # When False, we ignore wallet_names regardless of what it is. + self.requires_wallet = False self.set_test_params() assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes if self.options.timeout_factor == 0 : @@ -186,15 +187,30 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') group = parser.add_mutually_exclusive_group() - group.add_argument("--descriptors", default=False, action="store_true", + group.add_argument("--descriptors", action='store_const', const=True, help="Run test using a descriptor wallet", dest='descriptors') - group.add_argument("--legacy-wallet", default=False, action="store_false", + group.add_argument("--legacy-wallet", action='store_const', const=False, help="Run test using legacy wallets", dest='descriptors') self.add_options(parser) self.options = parser.parse_args() self.options.previous_releases_path = previous_releases_path + config = configparser.ConfigParser() + config.read_file(open(self.options.configfile)) + self.config = config + + if self.options.descriptors is None: + # Prefer BDB unless it isn't available + if self.is_bdb_compiled(): + self.options.descriptors = False + elif self.is_sqlite_compiled(): + self.options.descriptors = True + else: + # If neither are compiled, tests requiring a wallet will be skipped and the value of self.options.descriptors won't matter + # It still needs to exist and be None in order for tests to work however. + self.options.descriptors = None + def setup(self): """Call this method to start up the test framework object with options set.""" @@ -204,9 +220,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.cachedir = os.path.abspath(self.options.cachedir) - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - self.config = config + config = self.config + fname_bitcoind = os.path.join( config["environment"]["BUILDDIR"], "src", @@ -379,7 +394,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = self.extra_args self.add_nodes(self.num_nodes, extra_args) self.start_nodes() - if self.is_wallet_compiled(): + if self.requires_wallet: self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: @@ -771,10 +786,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def skip_if_no_wallet(self): """Skip the running test if wallet has not been compiled.""" + self.requires_wallet = True if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") if self.options.descriptors: self.skip_if_no_sqlite() + else: + self.skip_if_no_bdb() def skip_if_no_sqlite(self): """Skip the running test if sqlite has not been compiled.""" diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index b3eb2d61a7..123c48852c 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -342,16 +342,25 @@ def initialize_datadir(dirname, n, chain): datadir = get_datadir_path(dirname, n) if not os.path.isdir(datadir): os.makedirs(datadir) - # Translate chain name to config name + write_config(os.path.join(datadir, "bitcoin.conf"), n=n, chain=chain) + os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) + os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) + return datadir + + +def write_config(config_path, *, n, chain, extra_config=""): + # Translate chain subdirectory name to config name if chain == 'testnet3': chain_name_conf_arg = 'testnet' chain_name_conf_section = 'test' else: chain_name_conf_arg = chain chain_name_conf_section = chain - with open(os.path.join(datadir, "bitcoin.conf"), 'w', encoding='utf8') as f: - f.write("{}=1\n".format(chain_name_conf_arg)) - f.write("[{}]\n".format(chain_name_conf_section)) + with open(config_path, 'w', encoding='utf8') as f: + if chain_name_conf_arg: + f.write("{}=1\n".format(chain_name_conf_arg)) + if chain_name_conf_section: + f.write("[{}]\n".format(chain_name_conf_section)) f.write("port=" + str(p2p_port(n)) + "\n") f.write("rpcport=" + str(rpc_port(n)) + "\n") f.write("fallbackfee=0.0002\n") @@ -364,9 +373,7 @@ def initialize_datadir(dirname, n, chain): f.write("upnp=0\n") f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") - os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) - os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) - return datadir + f.write(extra_config) def get_datadir_path(dirname, n): diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 7cb74bdcb3..edd7792608 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -76,4 +76,4 @@ class MiniWallet: from_node.sendrawtransaction(tx_hex) assert_equal(tx_info['vsize'], vsize) assert_equal(tx_info['fees']['base'], fee) - return {'txid': tx_info['txid'], 'wtxid': from_node.decoderawtransaction(tx_hex)['hash'], 'hex': tx_hex} + return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex} diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9bbf862568..1e7d9e4b0d 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -86,29 +86,29 @@ EXTENDED_SCRIPTS = [ BASE_SCRIPTS = [ # Scripts that are run by default. # Longest test should go first, to favor running tests in parallel - 'wallet_hd.py', + 'wallet_hd.py --legacy-wallet', 'wallet_hd.py --descriptors', - 'wallet_backup.py', + 'wallet_backup.py --legacy-wallet', 'wallet_backup.py --descriptors', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', 'feature_maxuploadtarget.py', 'feature_block.py', - 'rpc_fundrawtransaction.py', + 'rpc_fundrawtransaction.py --legacy-wallet', 'rpc_fundrawtransaction.py --descriptors', 'p2p_compactblocks.py', 'feature_segwit.py --legacy-wallet', # vv Tests less than 2m vv - 'wallet_basic.py', + 'wallet_basic.py --legacy-wallet', 'wallet_basic.py --descriptors', - 'wallet_labels.py', + 'wallet_labels.py --legacy-wallet', 'wallet_labels.py --descriptors', 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'wallet_listtransactions.py', + 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', 'feature_taproot.py', # vv Tests less than 60s vv @@ -116,28 +116,29 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', - 'wallet_listreceivedby.py', + 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', - 'wallet_abandonconflict.py', + 'wallet_abandonconflict.py --legacy-wallet', 'wallet_abandonconflict.py --descriptors', 'feature_csv_activation.py', - 'rpc_rawtransaction.py', + 'rpc_rawtransaction.py --legacy-wallet', 'rpc_rawtransaction.py --descriptors', - 'wallet_address_types.py', + 'wallet_address_types.py --legacy-wallet', 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', 'p2p_feefilter.py', 'feature_reindex.py', 'feature_abortnode.py', # vv Tests less than 30s vv - 'wallet_keypool_topup.py', + 'wallet_keypool_topup.py --legacy-wallet', 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', + 'rpc_invalid_address_message.py', 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', - 'tool_wallet.py', + 'tool_wallet.py --legacy-wallet', 'tool_wallet.py --descriptors', 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', @@ -145,14 +146,14 @@ BASE_SCRIPTS = [ 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', - 'wallet_avoidreuse.py', + 'wallet_avoidreuse.py --legacy-wallet', 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', - 'wallet_multiwallet.py', + 'wallet_multiwallet.py --legacy-wallet', 'wallet_multiwallet.py --descriptors', 'wallet_multiwallet.py --usecli', - 'wallet_createwallet.py', + 'wallet_createwallet.py --legacy-wallet', 'wallet_createwallet.py --usecli', 'wallet_createwallet.py --descriptors', 'wallet_watchonly.py --legacy-wallet', @@ -160,27 +161,27 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', - 'rpc_psbt.py', + 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', - 'rpc_signrawtransaction.py', + 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', - 'wallet_groups.py', + 'wallet_groups.py --legacy-wallet', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', 'p2p_disconnect_ban.py', 'rpc_decodescript.py', 'rpc_blockchain.py', 'rpc_deprecated.py', - 'wallet_disable.py', + 'wallet_disable.py --legacy-wallet', 'wallet_disable.py --descriptors', 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', 'rpc_net.py', - 'wallet_keypool.py', + 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', 'p2p_nobloomfilter_messages.py', @@ -194,9 +195,9 @@ BASE_SCRIPTS = [ 'p2p_invalid_tx.py', 'feature_assumevalid.py', 'example_test.py', - 'wallet_txn_doublespend.py', + 'wallet_txn_doublespend.py --legacy-wallet', 'wallet_txn_doublespend.py --descriptors', - 'feature_backwards_compatibility.py', + 'feature_backwards_compatibility.py --legacy-wallet', 'feature_backwards_compatibility.py --descriptors', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', @@ -205,58 +206,60 @@ BASE_SCRIPTS = [ 'feature_rbf.py', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py', + 'rpc_createmultisig.py --legacy-wallet', 'rpc_createmultisig.py --descriptors', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', - 'wallet_importprunedfunds.py', + 'wallet_importprunedfunds.py --legacy-wallet', 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', 'rpc_signmessage.py', 'rpc_generateblock.py', 'rpc_generate.py', - 'wallet_balance.py', + 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', - 'feature_nulldummy.py', + 'feature_nulldummy.py --legacy-wallet', 'feature_nulldummy.py --descriptors', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py --legacy-wallet', 'wallet_import_with_label.py --legacy-wallet', 'wallet_importdescriptors.py --descriptors', - 'wallet_upgradewallet.py', + 'wallet_upgradewallet.py --legacy-wallet', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', 'mining_basic.py', 'feature_signet.py', - 'wallet_bumpfee.py', + 'wallet_bumpfee.py --legacy-wallet', 'wallet_bumpfee.py --descriptors', 'wallet_implicitsegwit.py --legacy-wallet', 'rpc_named_arguments.py', - 'wallet_listsinceblock.py', + 'wallet_listsinceblock.py --legacy-wallet', 'wallet_listsinceblock.py --descriptors', + 'wallet_listdescriptors.py --descriptors', 'p2p_leak.py', - 'wallet_encryption.py', + 'wallet_encryption.py --legacy-wallet', 'wallet_encryption.py --descriptors', 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', - 'wallet_resendwallettransactions.py', + 'wallet_resendwallettransactions.py --legacy-wallet', 'wallet_resendwallettransactions.py --descriptors', - 'wallet_fallbackfee.py', + 'wallet_fallbackfee.py --legacy-wallet', 'wallet_fallbackfee.py --descriptors', 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', - 'wallet_create_tx.py', - 'wallet_send.py', + 'wallet_create_tx.py --legacy-wallet', + 'wallet_send.py --legacy-wallet', + 'wallet_send.py --descriptors', 'wallet_create_tx.py --descriptors', 'p2p_fingerprint.py', 'feature_uacomment.py', - 'wallet_coinbase_category.py', + 'wallet_coinbase_category.py --legacy-wallet', 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', 'feature_loadblock.py', @@ -264,6 +267,7 @@ BASE_SCRIPTS = [ 'p2p_add_connections.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', + 'p2p_message_capture.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 8a1af24dcf..28103793df 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -183,11 +183,13 @@ class ToolWalletTest(BitcoinTestFramework): def test_invalid_tool_commands_and_args(self): self.log.info('Testing that various invalid commands raise with specific error messages') - self.assert_raises_tool_error('Invalid command: foo', 'foo') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'foo'", 'foo') # `bitcoin-wallet help` raises an error. Use `bitcoin-wallet -help`. - self.assert_raises_tool_error('Invalid command: help', 'help') - self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'help'", 'help') + self.assert_raises_tool_error('Error: Additional arguments provided (create). Methods do not take arguments. Please refer to `-help`.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('No method provided. Run `bitcoin-wallet -help` for valid methods.') + self.assert_raises_tool_error('Wallet name must be provided when creating a new wallet.', 'create') locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: @@ -348,7 +350,8 @@ class ToolWalletTest(BitcoinTestFramework): self.log.info('Checking createfromdump') self.do_tool_createfromdump("load", "wallet.dump") - self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") + if self.is_bdb_compiled(): + self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") if self.is_sqlite_compiled(): self.do_tool_createfromdump("load-sqlite", "wallet.dump", "sqlite") diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 229c134a4b..bc4fa90e83 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -65,7 +65,6 @@ def assert_balances(node, mine): class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 6bcb03e8ed..4a589f0393 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -586,7 +586,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) # Test getaddressinfo on external address. Note that these addresses are taken from disablewallet.py - assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") address_info = self.nodes[0].getaddressinfo("mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info['address'], "mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info["scriptPubKey"], "76a9144e3854046c7bd1594ac904e4793b6a45b36dea0988ac") diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c8c1f2e374..5fc8438e8f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -425,6 +425,9 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): original_txid = watcher.sendrawtransaction(psbt_final["hex"]) assert_equal(len(watcher.decodepsbt(psbt)["tx"]["vin"]), 1) + # bumpfee can't be used on watchonly wallets + assert_raises_rpc_error(-4, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead.", watcher.bumpfee, original_txid) + # Bump fee, obnoxiously high to add additional watchonly input bumped_psbt = watcher.psbtbumpfee(original_txid, {"fee_rate": HIGH}) assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index cf3317121f..16a0a50b07 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -17,7 +17,6 @@ from test_framework.wallet_util import bytes_to_wif, generate_wif_key class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 1de41a5f96..c04a15a67c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -23,11 +23,14 @@ class WalletDescriptorTest(BitcoinTestFramework): self.skip_if_no_sqlite() def run_test(self): - # Make a legacy wallet and check it is BDB - self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) - wallet_info = self.nodes[0].getwalletinfo() - assert_equal(wallet_info['format'], 'bdb') - self.nodes[0].unloadwallet("legacy1") + if self.is_bdb_compiled(): + # Make a legacy wallet and check it is BDB + self.nodes[0].createwallet(wallet_name="legacy1", descriptors=False) + wallet_info = self.nodes[0].getwalletinfo() + assert_equal(wallet_info['format'], 'bdb') + self.nodes[0].unloadwallet("legacy1") + else: + self.log.warning("Skipping BDB test") # Make a descriptor wallet self.log.info("Making a descriptor wallet") diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py new file mode 100755 index 0000000000..9f8c341bc7 --- /dev/null +++ b/test/functional/wallet_listdescriptors.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the listdescriptors RPC.""" + +from test_framework.descriptors import ( + descsum_create +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class ListDescriptorsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + # do not create any wallet by default + def init_wallet(self, i): + return + + def run_test(self): + node = self.nodes[0] + assert_raises_rpc_error(-18, 'No wallet is loaded.', node.listdescriptors) + + self.log.info('Test that the command is not available for legacy wallets.') + node.createwallet(wallet_name='w1', descriptors=False) + assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) + + self.log.info('Test the command for empty descriptors wallet.') + node.createwallet(wallet_name='w2', blank=True, descriptors=True) + assert_equal(0, len(node.get_wallet_rpc('w2').listdescriptors())) + + self.log.info('Test the command for a default descriptors wallet.') + node.createwallet(wallet_name='w3', descriptors=True) + result = node.get_wallet_rpc('w3').listdescriptors() + assert_equal(6, len(result)) + assert_equal(6, len([d for d in result if d['active']])) + assert_equal(3, len([d for d in result if d['internal']])) + for item in result: + assert item['desc'] != '' + assert item['next'] == 0 + assert item['range'] == [0, 0] + assert item['timestamp'] is not None + + self.log.info('Test non-active non-range combo descriptor') + node.createwallet(wallet_name='w4', blank=True, descriptors=True) + wallet = node.get_wallet_rpc('w4') + wallet.importdescriptors([{ + 'desc': descsum_create('combo(' + node.get_deterministic_priv_key().key + ')'), + 'timestamp': 1296688602, + }]) + expected = [{'active': False, + 'desc': 'combo(0227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3f)#np574htj', + 'timestamp': 1296688602}] + assert_equal(expected, wallet.listdescriptors()) + + +if __name__ == '__main__': + ListDescriptorsTest().main() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index bb89e76a9a..bf24b9c7b3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -304,12 +304,12 @@ class MultiWalletTest(BitcoinTestFramework): if self.options.descriptors: assert_raises_rpc_error(-4, "Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?", self.nodes[0].loadwallet, wallet_names[0]) else: - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) # This tests the default wallet that BDB makes, so SQLite wallet doesn't need to test this # Fail to load duplicate wallets by different ways (directory and filepath) path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary # Fail to load if one wallet is a copy of another diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 9835c5a2af..880341fdd9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -8,6 +8,7 @@ from decimal import Decimal, getcontext from itertools import product from test_framework.authproxy import JSONRPCException +from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -168,49 +169,91 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name="w1") w1 = self.nodes[1].get_wallet_rpc("w1") # w2 contains the private keys for w3 - self.nodes[1].createwallet(wallet_name="w2") + self.nodes[1].createwallet(wallet_name="w2", blank=True) w2 = self.nodes[1].get_wallet_rpc("w2") + xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" + xpub = "tpubD6NzVbkrYhZ4YkEfMbRJkQyZe7wTkbTNRECozCtJPtdLRn6cT1QKb8yHjwAPcAr26eHBFYs5iLiFFnCbwPRsncCKUKCfubHDMGKzMVcN1Jg" + if self.options.descriptors: + w2.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpriv + "/0/0/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True + },{ + "desc": descsum_create("wpkh(" + xpriv + "/0/1/*)"), + "timestamp": "now", + "range": [0, 100], + "active": True, + "internal": True + }]) + else: + w2.sethdseed(True) + # w3 is a watch-only wallet, based on w2 self.nodes[1].createwallet(wallet_name="w3", disable_private_keys=True) w3 = self.nodes[1].get_wallet_rpc("w3") - for _ in range(3): - a2_receive = w2.getnewaddress() - a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation - res = w3.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], + if self.options.descriptors: + # Match the privkeys in w2 for descriptors + res = w3.importdescriptors([{ + "desc": descsum_create("wpkh(" + xpub + "/0/0/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "watchonly": True },{ - "desc": w2.getaddressinfo(a2_change)["desc"], + "desc": descsum_create("wpkh(" + xpub + "/0/1/*)"), "timestamp": "now", + "range": [0, 100], "keypool": True, + "active": True, "internal": True, "watchonly": True }]) assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w3 - self.nodes[0].generate(1) - self.sync_blocks() - - # w4 has private keys enabled, but only contains watch-only keys (from w2) - self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) - w4 = self.nodes[1].get_wallet_rpc("w4") for _ in range(3): a2_receive = w2.getnewaddress() - res = w4.importmulti([{ - "desc": w2.getaddressinfo(a2_receive)["desc"], - "timestamp": "now", - "keypool": False, - "watchonly": True - }]) - assert_equal(res, [{"success": True}]) + if not self.options.descriptors: + # Because legacy wallets use exclusively hardened derivation, we can't do a ranged import like we do for descriptors + a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation + res = w3.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": True, + "watchonly": True + },{ + "desc": w2.getaddressinfo(a2_change)["desc"], + "timestamp": "now", + "keypool": True, + "internal": True, + "watchonly": True + }]) + assert_equal(res, [{"success": True}, {"success": True}]) - w0.sendtoaddress(a2_receive, 10) # fund w4 + w0.sendtoaddress(a2_receive, 10) # fund w3 self.nodes[0].generate(1) self.sync_blocks() + if not self.options.descriptors: + # w4 has private keys enabled, but only contains watch-only keys (from w2) + # This is legacy wallet behavior only as descriptor wallets don't allow watchonly and non-watchonly things in the same wallet. + self.nodes[1].createwallet(wallet_name="w4", disable_private_keys=False) + w4 = self.nodes[1].get_wallet_rpc("w4") + for _ in range(3): + a2_receive = w2.getnewaddress() + res = w4.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": False, + "watchonly": True + }]) + assert_equal(res, [{"success": True}]) + + w0.sendtoaddress(a2_receive, 10) # fund w4 + self.nodes[0].generate(1) + self.sync_blocks() + self.log.info("Send to address...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=True) @@ -241,11 +284,15 @@ class WalletSendTest(BitcoinTestFramework): res = w2.walletprocesspsbt(res["psbt"]) assert res["complete"] - self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") - self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) - res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) - res = w2.walletprocesspsbt(res["psbt"]) - assert res["complete"] + if not self.options.descriptors: + # Descriptor wallets do not allow mixed watch-only and non-watch-only things in the same wallet. + # This is specifically testing that w4 ignores its own private keys and creates a psbt with send + # which is not something that needs to be tested in descriptor wallets. + self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") + self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) + res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] self.log.info("Create OP_RETURN...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index d0bb6135a8..fbc0f995d2 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -57,6 +57,7 @@ class UpgradeWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_bdb() self.skip_if_no_previous_releases() def setup_network(self): diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index b0c41b2738..24799fe5f2 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -14,7 +14,6 @@ from test_framework.util import ( class CreateWalletWatchonlyTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 3c743603bb..611061072f 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -14,6 +14,16 @@ import subprocess import sys +def get_fuzz_env(*, target, source_dir): + return { + 'FUZZ': target, + 'UBSAN_OPTIONS': + f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'ASAN_OPTIONS': # symbolizer disabled due to https://github.com/google/sanitizers/issues/1364#issuecomment-761072085 + 'symbolize=0:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1', + } + + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -129,9 +139,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), '-help=1', ], - env={ - 'FUZZ': test_list_selection[0] - }, + env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), timeout=20, check=True, stderr=subprocess.PIPE, @@ -148,6 +156,7 @@ def main(): if args.generate: return generate_corpus_seeds( fuzz_pool=fuzz_pool, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], seed_dir=args.seed_dir, targets=test_list_selection, @@ -158,6 +167,7 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], merge_dir=args.m_dir, ) @@ -167,12 +177,13 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], use_valgrind=args.valgrind, ) -def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): +def generate_corpus_seeds(*, fuzz_pool, src_dir, build_dir, seed_dir, targets): """Generates new corpus seeds. Run {targets} without input, and outputs the generated corpus seeds to @@ -186,9 +197,7 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): ' '.join(command), subprocess.run( command, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -209,13 +218,15 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): future.result() -def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir): logging.info("Merge the inputs from the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), '-merge=1', + '-shuffle=0', + '-prefer_small=1', '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), os.path.join(merge_dir, t), @@ -227,9 +238,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): output = 'Run {} with args {}\n'.format(t, " ".join(args)) output += subprocess.run( args, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -242,7 +251,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): future.result() -def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) @@ -257,7 +266,12 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): def job(t, args): output = 'Run {} with args {}'.format(t, args) - result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True) + result = subprocess.run( + args, + env=get_fuzz_env(target=t, source_dir=src_dir), + stderr=subprocess.PIPE, + universal_newlines=True, + ) output += result.stderr return output, result diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 1348b8246b..b177fbb4b2 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -175,6 +175,7 @@ def check_host(args) -> int: './depends/config.guess').decode()) if args.download_binary: platforms = { + 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', 'x86_64-apple-darwin*': 'osx64', } diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py deleted file mode 100755 index 0a4cc875d0..0000000000 --- a/test/lint/check-rpc-mappings.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Check RPC argument consistency.""" - -from collections import defaultdict -import os -import re -import sys - -# Source files (relative to root) to scan for dispatch tables -SOURCES = [ - "src/rpc/server.cpp", - "src/rpc/blockchain.cpp", - "src/rpc/mining.cpp", - "src/rpc/misc.cpp", - "src/rpc/net.cpp", - "src/rpc/rawtransaction.cpp", - "src/wallet/rpcwallet.cpp", -] -# Source file (relative to root) containing conversion mapping -SOURCE_CLIENT = 'src/rpc/client.cpp' -# Argument names that should be ignored in consistency checks -IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'} - -class RPCCommand: - def __init__(self, name, args): - self.name = name - self.args = args - -class RPCArgument: - def __init__(self, names, idx): - self.names = names - self.idx = idx - self.convert = False - -def parse_string(s): - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - -def process_commands(fname): - """Find and parse dispatch table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if re.match(r"static const CRPCCommand .*\[\] =", line): - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(2)) - args_str = m.group(4).strip() - if args_str: - args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))] - else: - args = [] - cmds.append(RPCCommand(name, args)) - assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps" - return cmds - -def process_mapping(fname): - """Find and parse conversion table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if line == 'static const CRPCConvertParam vRPCConvertParams[] =': - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(1)) - idx = int(m.group(2)) - argname = parse_string(m.group(3)) - cmds.append((name, idx, argname)) - assert not in_rpcs and cmds - return cmds - -def main(): - if len(sys.argv) != 2: - print('Usage: {} ROOT-DIR'.format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - root = sys.argv[1] - - # Get all commands from dispatch tables - cmds = [] - for fname in SOURCES: - cmds += process_commands(os.path.join(root, fname)) - - cmds_by_name = {} - for cmd in cmds: - cmds_by_name[cmd.name] = cmd - - # Get current convert mapping for client - client = SOURCE_CLIENT - mapping = set(process_mapping(os.path.join(root, client))) - - print('* Checking consistency between dispatch tables and vRPCConvertParams') - - # Check mapping consistency - errors = 0 - for (cmdname, argidx, argname) in mapping: - try: - rargnames = cmds_by_name[cmdname].args[argidx].names - except IndexError: - print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname)) - errors += 1 - continue - if argname not in rargnames: - print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr) - errors += 1 - - # Check for conflicts in vRPCConvertParams conversion - # All aliases for an argument must either be present in the - # conversion table, or not. Anything in between means an oversight - # and some aliases won't work. - for cmd in cmds: - for arg in cmd.args: - convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names] - if any(convert) != all(convert): - print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert)) - errors += 1 - arg.convert = all(convert) - - # Check for conversion difference by argument name. - # It is preferable for API consistency that arguments with the same name - # have the same conversion, so bin by argument name. - all_methods_by_argname = defaultdict(list) - converts_by_argname = defaultdict(list) - for cmd in cmds: - for arg in cmd.args: - for argname in arg.names: - all_methods_by_argname[argname].append(cmd.name) - converts_by_argname[argname].append(arg.convert) - - for argname, convert in converts_by_argname.items(): - if all(convert) != any(convert): - if argname in IGNORE_DUMMY_ARGS: - # these are testing or dummy, don't warn for them - continue - print('WARNING: conversion mismatch for argument named %s (%s)' % - (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) - - sys.exit(errors > 0) - - -if __name__ == '__main__': - main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index dc032665e4..fc0b86a297 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -60,17 +60,13 @@ EXPECTED_BOOST_INCLUDES=( boost/multi_index/ordered_index.hpp boost/multi_index/sequenced_index.hpp boost/multi_index_container.hpp - boost/preprocessor/cat.hpp - boost/preprocessor/stringize.hpp boost/process.hpp boost/signals2/connection.hpp boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp - boost/thread/condition_variable.hpp - boost/thread/mutex.hpp + boost/thread/lock_types.hpp boost/thread/shared_mutex.hpp - boost/thread/thread.hpp ) for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index c800fd20db..37fcf7804a 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -33,7 +33,7 @@ if [ -z "${COMMIT_RANGE}" ]; then fi showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then + if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then echo "Failed to get a diff" exit 1 fi diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 9a52cd4b57..97f0f45e7f 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,3 +1,7 @@ +# -fsanitize=undefined suppressions +# ================================= +# No suppressions at the moment. Hooray! + # -fsanitize=integer suppressions # =============================== # Unsigned integer overflow occurs when the result of an unsigned integer @@ -6,7 +10,8 @@ # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. -unsigned-integer-overflow:*/include/c++/*/bits/basic_string.tcc +unsigned-integer-overflow:*/include/c++/ +unsigned-integer-overflow:addrman.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -15,34 +20,41 @@ unsigned-integer-overflow:bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h +unsigned-integer-overflow:coins.cpp +unsigned-integer-overflow:compressor.cpp unsigned-integer-overflow:core_write.cpp -unsigned-integer-overflow:crypto/* +unsigned-integer-overflow:crypto/ +# unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange +unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:leveldb/db/log_reader.cc -unsigned-integer-overflow:leveldb/util/bloom.cc -unsigned-integer-overflow:leveldb/util/crc32c.h -unsigned-integer-overflow:leveldb/util/hash.cc +unsigned-integer-overflow:leveldb/ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h +unsigned-integer-overflow:pubkey.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp - -implicit-integer-sign-change:*/include/c++/*/bits/*.h +implicit-integer-sign-change:*/include/boost/ +implicit-integer-sign-change:*/include/c++/ implicit-integer-sign-change:*/new_allocator.h -implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp +implicit-integer-sign-change:addrman.h implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp implicit-integer-sign-change:bloom.cpp -implicit-integer-sign-change:chain.* +implicit-integer-sign-change:chain.cpp +implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h -implicit-integer-sign-change:crypto/* +implicit-integer-sign-change:crc32c/ +implicit-integer-sign-change:crypto/ +# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange +implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:key.cpp implicit-integer-sign-change:noui.cpp +implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h implicit-integer-sign-change:protocol.cpp implicit-integer-sign-change:script/bitcoinconsensus.cpp @@ -53,24 +65,38 @@ implicit-integer-sign-change:test/coins_tests.cpp implicit-integer-sign-change:test/pow_tests.cpp implicit-integer-sign-change:test/prevector_tests.cpp implicit-integer-sign-change:test/sighash_tests.cpp +implicit-integer-sign-change:test/skiplist_tests.cpp implicit-integer-sign-change:test/streams_tests.cpp implicit-integer-sign-change:test/transaction_tests.cpp implicit-integer-sign-change:txmempool.cpp -implicit-integer-sign-change:util/strencodings.* +implicit-integer-sign-change:util/strencodings.cpp +implicit-integer-sign-change:util/strencodings.h implicit-integer-sign-change:validation.cpp implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp +implicit-signed-integer-truncation:addrman.cpp +implicit-signed-integer-truncation:addrman.h implicit-signed-integer-truncation:chain.h -implicit-signed-integer-truncation:crypto/* +implicit-signed-integer-truncation:crypto/ implicit-signed-integer-truncation:cuckoocache.h -implicit-signed-integer-truncation:leveldb/* +implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:net.cpp +implicit-signed-integer-truncation:net_processing.cpp implicit-signed-integer-truncation:streams.h implicit-signed-integer-truncation:test/arith_uint256_tests.cpp implicit-signed-integer-truncation:test/skiplist_tests.cpp implicit-signed-integer-truncation:torcontrol.cpp -implicit-unsigned-integer-truncation:crypto/* -implicit-unsigned-integer-truncation:leveldb/* +implicit-unsigned-integer-truncation:*/include/c++/ +implicit-unsigned-integer-truncation:crypto/ +implicit-unsigned-integer-truncation:leveldb/ # std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage -implicit-integer-sign-change:crc32c/* +shift-base:*/include/c++/ +shift-base:arith_uint256.cpp +shift-base:crypto/ +shift-base:hash.cpp +shift-base:leveldb/ +shift-base:net_processing.cpp +shift-base:streams.h +shift-base:util/bip32.cpp |