diff options
92 files changed, 1293 insertions, 742 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b3d58461e0..8b88262e51 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -113,7 +113,7 @@ task: << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal - cpu: 4 # Double CPU and increase Memory to avoid timeout + cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: MAKEJOBS: "-j8" @@ -168,13 +168,13 @@ task: FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: - name: 'macOS 10.15 native [gui] [no depends]' + name: 'macOS 11 native [gui] [no depends]' macos_brew_addon_script: - brew install boost libevent berkeley-db4 qt miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt << : *GLOBAL_TASK_TEMPLATE osx_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: catalina-xcode-12.1 # https://cirrus-ci.org/guide/macOS + image: big-sur-xcode-12.4 # https://cirrus-ci.org/guide/macOS env: DANGER_RUN_CI_ON_HOST: "true" CI_USE_APT_INSTALL: "no" diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index cce06e2fff..57b4c10d77 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -108,11 +108,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: - dnl For Qt5, we can check a header to find out whether Qt is build - dnl statically. When Qt is built statically, some plugins must be linked into - dnl the final binary as well. - dnl _BITCOIN_QT_CHECK_STATIC_PLUGIN does a quick link-check and appends the - dnl results to QT_LIBS. + dnl We check a header to find out whether Qt is built statically. + dnl When Qt is built statically, some plugins must be linked into + dnl the final binary as well. _BITCOIN_QT_CHECK_STATIC_PLUGIN does + dnl a quick link-check and appends the results to QT_LIBS. BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS @@ -162,7 +161,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) elif test "x$TARGET_OS" = xandroid; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS" + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype $QT_LIBS" AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) fi fi @@ -171,7 +170,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ ]) if test "x$qt_bin_path" = x; then - qt_bin_path="`$PKG_CONFIG --variable=host_bins Qt5Core 2>/dev/null`" + qt_bin_path="`$PKG_CONFIG --variable=host_bins ${qt_lib_prefix}Core 2>/dev/null`" fi if test "x$use_hardening" != xno; then @@ -282,12 +281,13 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_SUBST(MOC_DEFS) ]) -dnl All macros below are internal and should _not_ be used from the main -dnl configure.ac. -dnl ---- +dnl All macros below are internal and should _not_ be used from configure.ac. -dnl Internal. Check if the linked version of Qt was built as static libs. -dnl Requires: Qt5. +dnl Internal. Check if the linked version of Qt was built statically. +dnl +dnl _BITCOIN_QT_IS_STATIC +dnl --------------------- +dnl dnl Requires: INCLUDES and LIBS must be populated as necessary. dnl Output: bitcoin_cv_static_qt=yes|no AC_DEFUN([_BITCOIN_QT_IS_STATIC],[ @@ -335,47 +335,54 @@ dnl dnl _BITCOIN_QT_CHECK_STATIC_LIBS dnl ----------------------------- dnl -dnl Inputs: no inputs. dnl Outputs: QT_LIBS is prepended. AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="-lQt5FontDatabaseSupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport${qt_lib_suffix}], [QT_LIBS="-lQt5EventDispatcherSupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport${qt_lib_suffix}], [QT_LIBS="-lQt5ThemeSupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport${qt_lib_suffix}], [QT_LIBS="-lQt5DeviceDiscoverySupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport${qt_lib_suffix}], [QT_LIBS="-lQt5AccessibilitySupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport${qt_lib_suffix}], [QT_LIBS="-lQt5FbSupport${qt_lib_suffix} $QT_LIBS"]) + PKG_CHECK_MODULES([QT_ACCESSIBILITY], [${qt_lib_prefix}AccessibilitySupport${qt_lib_suffix}], [QT_LIBS="$QT_ACCESSIBILITY_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_DEVICEDISCOVERY], [${qt_lib_prefix}DeviceDiscoverySupport${qt_lib_suffix}], [QT_LIBS="$QT_DEVICEDISCOVERY_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_EDID], [${qt_lib_prefix}EdidSupport${qt_lib_suffix}], [QT_LIBS="$QT_EDID_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_EVENTDISPATCHER], [${qt_lib_prefix}EventDispatcherSupport${qt_lib_suffix}], [QT_LIBS="$QT_EVENTDISPATCHER_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_FB], [${qt_lib_prefix}FbSupport${qt_lib_suffix}], [QT_LIBS="$QT_FB_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport${qt_lib_suffix}], [QT_LIBS="-lQt5ClipboardSupport${qt_lib_suffix} $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport${qt_lib_suffix}], [QT_LIBS="-lQt5GraphicsSupport${qt_lib_suffix} $QT_LIBS"]) + PKG_CHECK_MODULES([QT_CLIPBOARD], [${qt_lib_prefix}ClipboardSupport${qt_lib_suffix}], [QT_LIBS="$QT_CLIPBOARD_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_GRAPHICS], [${qt_lib_prefix}GraphicsSupport${qt_lib_suffix}], [QT_LIBS="$QT_GRAPHICS_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xwindows; then - PKG_CHECK_MODULES([QTWINDOWSUIAUTOMATION], [Qt5WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="-lQt5WindowsUIAutomationSupport${qt_lib_suffix} $QT_LIBS"]) + PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xandroid; then + PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) fi ]) dnl Internal. Find Qt libraries using pkg-config. +dnl +dnl _BITCOIN_QT_FIND_LIBS +dnl --------------------- +dnl dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core${qt_lib_suffix} $qt_version], [], + PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_CORE_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_CORE_LIBS $QT_LIBS"], [BITCOIN_QT_FAIL([${qt_lib_prefix}Core${qt_lib_suffix} $qt_version not found])]) ]) BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui${qt_lib_suffix} $qt_version], [], + PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_GUI_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_GUI_LIBS $QT_LIBS"], [BITCOIN_QT_FAIL([${qt_lib_prefix}Gui${qt_lib_suffix} $qt_version not found])]) ]) BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets${qt_lib_suffix} $qt_version], [], + PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_WIDGETS_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_WIDGETS_LIBS $QT_LIBS"], [BITCOIN_QT_FAIL([${qt_lib_prefix}Widgets${qt_lib_suffix} $qt_version not found])]) ]) BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network${qt_lib_suffix} $qt_version], [], + PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network${qt_lib_suffix} $qt_version], [QT_INCLUDES="$QT_NETWORK_CFLAGS $QT_INCLUDES" QT_LIBS="$QT_NETWORK_LIBS $QT_LIBS"], [BITCOIN_QT_FAIL([${qt_lib_prefix}Network${qt_lib_suffix} $qt_version not found])]) ]) - QT_INCLUDES="$QT_CORE_CFLAGS $QT_GUI_CFLAGS $QT_WIDGETS_CFLAGS $QT_NETWORK_CFLAGS" - QT_LIBS="$QT_CORE_LIBS $QT_GUI_LIBS $QT_WIDGETS_LIBS $QT_NETWORK_LIBS" BITCOIN_QT_CHECK([ PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test${qt_lib_suffix} $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) diff --git a/build-aux/m4/l_socket.m4 b/build-aux/m4/l_socket.m4 new file mode 100644 index 0000000000..38923a98fc --- /dev/null +++ b/build-aux/m4/l_socket.m4 @@ -0,0 +1,36 @@ +# Illumos/SmartOS requires linking with -lsocket if +# using getifaddrs & freeifaddrs + +m4_define([_CHECK_SOCKET_testbody], [[ + #include <sys/types.h> + #include <ifaddrs.h> + + int main() { + struct ifaddrs *ifaddr; + getifaddrs(&ifaddr); + freeifaddrs(ifaddr); + } +]]) + +AC_DEFUN([CHECK_SOCKET], [ + + AC_LANG_PUSH(C++) + + AC_MSG_CHECKING([whether ifaddrs funcs can be used without link library]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + LIBS="$LIBS -lsocket" + AC_MSG_CHECKING([whether getifaddrs needs -lsocket]) + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_FAILURE([cannot figure out how to use getifaddrs]) + ]) + ]) + + AC_LANG_POP +]) diff --git a/configure.ac b/configure.ac index 9ac2643374..a53705aa22 100644 --- a/configure.ac +++ b/configure.ac @@ -472,7 +472,6 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi @@ -936,7 +935,7 @@ fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) -AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, +AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include <sys/types.h> #include <ifaddrs.h>] ) diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 7ee7f056d9..bc5535b4c7 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -87,7 +87,7 @@ Files: src/qt/res/icons/proxy.png Copyright: Cristian Mircea Messel License: public-domain -Files: src/qt/fonts/RobotoMono-Bold.ttf +Files: src/qt/res/fonts/RobotoMono-Bold.ttf License: Apache-2.0 Comment: Site: https://fonts.google.com/specimen/Roboto+Mono diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME index a2ca768aaa..330fbc041b 100755 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -4,7 +4,7 @@ # # A helper script to be sourced into the gitian descriptors -if RECENT_TAG="$(git describe --exact-match HEAD)"; then +if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then VERSION="${RECENT_TAG#v}" else VERSION="$(git rev-parse --short=12 HEAD)" diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 1122ec9ba5..2d128c7ed6 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -80,6 +80,50 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` +### Controlling the number of threads used by `guix` build commands + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that there is a +`--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + ## Usage ### As a Tool for Deterministic Builds @@ -125,12 +169,16 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). -* _**MAX_JOBS**_ +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` - Override the maximum number of jobs to run simultaneously, you might want to - do so on a memory-limited machine. This may be passed to `make` as in `make - --jobs="$MAX_JOBS"` or `xargs` as in `xargs -P"$MAX_JOBS"`. _(defaults to the - value of `nproc` outside the container)_ + _(defaults to the value of `nproc` outside the container)_ * _**SOURCE_DATE_EPOCH**_ @@ -217,6 +265,57 @@ To use dongcarl's substitute server for Bitcoin Core builds after having export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` +## Troubleshooting + +### Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion: + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild stil did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +#### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + ## FAQ ### How can I trust the binary installation? diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build index 7c32fb5726..da127b54aa 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build @@ -2,22 +2,26 @@ export LC_ALL=C set -e -o pipefail +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + ################### -## Sanity Checks ## +## SANITY CHECKS ## ################### ################ -# Check 1: Make sure that we can invoke required tools +# Required non-builtin commands should be invokable ################ -for cmd in git make guix cat mkdir curl; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi -done + +check_tools cat mkdir make git guix ################ -# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +# GUIX_BUILD_OPTIONS should be empty ################ # # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that @@ -45,8 +49,9 @@ exit 1 fi ################ -# Check 3: Make sure that we're not in a dirty worktree +# The git worktree should not be dirty ################ + if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then cat << EOF ERR: The current git worktree is dirty, which may lead to broken builds. @@ -60,12 +65,12 @@ Hint: To make your git worktree clean, You may want to: using a dirty worktree EOF exit 1 -else - GIT_COMMIT=$(git rev-parse --short=12 HEAD) fi +mkdir -p "$VERSION_BASE" + ################ -# Check 4: Make sure that build directories do not exist +# Build directories should not exist ################ # Default to building for all supported HOSTs (overridable by environment) @@ -73,14 +78,12 @@ export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu ri x86_64-w64-mingw32 x86_64-apple-darwin18}" -DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" - # Usage: distsrc_for_host HOST # # HOST: The current platform triple we're building for # distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" } # Accumulate a list of build directories that already exist... @@ -106,12 +109,11 @@ for host in $hosts_distsrc_exists; do done exit 1 else - mkdir -p "$DISTSRC_BASE" fi ################ -# Check 5: When building for darwin, make sure that the macOS SDK exists +# When building for darwin, the macOS SDK should exists ################ for host in $HOSTS; do @@ -128,13 +130,40 @@ for host in $HOSTS; do esac done +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + ######### -# Setup # +# SETUP # ######### # Determine the maximum number of jobs to run simultaneously (overridable by # environment) -MAX_JOBS="${MAX_JOBS:-$(nproc)}" +JOBS="${JOBS:-$(nproc)}" # Usage: host_to_commonname HOST # @@ -152,7 +181,7 @@ host_to_commonname() { # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do - make -C "${PWD}/depends" -j"$MAX_JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done # Determine the reference time used for determinism (overridable by environment) @@ -164,19 +193,29 @@ time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ + --fallback \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" } # Make sure an output directory exists for our builds -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: outdir_for_host HOST +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}" +} + ######### -# Build # +# BUILD # ######### # Function to be called when building for host ${1} and the user interrupts the @@ -216,15 +255,15 @@ for host in $HOSTS; do # shellcheck disable=SC2030 cat << EOF -INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${MAX_JOBS:?not set} jobs + ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '${OUTDIR:?not set}' - ...bind-mounted in container to: '/outdir' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build @@ -299,24 +338,28 @@ EOF --no-cwd \ --share="$PWD"=/bitcoin \ --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR"=/outdir \ + --share="$OUTDIR_BASE"=/outdir-base \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ ${SDK_PATH:+--share="$SDK_PATH"} \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ + --fallback \ + --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ + DISTNAME="$DISTNAME" \ + JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR=/outdir \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 051066a6a0..4239c3d475 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -24,9 +24,11 @@ fi # Check that required environment variables are set cat << EOF Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - MAX_JOBS: ${MAX_JOBS:?not set} + JOBS: ${JOBS:?not set} DISTSRC: ${DISTSRC:?not set} OUTDIR: ${OUTDIR:?not set} EOF @@ -52,16 +54,36 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" -export CPATH="${NATIVE_GCC}/include" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + +unset LIBRARY_PATH +unset CPATH unset C_INCLUDE_PATH unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + case "$HOST" in *darwin*) # When targeting darwin, zlib is required by native_libdmg-hfsplus. zlib_store_path=$(store_path "zlib") - export LIBRARY_PATH="${zlib_store_path}/lib:${LIBRARY_PATH}" - export CPATH="${zlib_store_path}/include:${CPATH}" + zlib_static_store_path=$(store_path "zlib" static) + + prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" + prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" esac # Set environment variables to point the CROSS toolchain to the right @@ -173,7 +195,7 @@ esac #################### # Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ +make -C depends --jobs="$JOBS" HOST="$HOST" \ ${V:+V=1} \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ @@ -198,11 +220,7 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ # Source Tarball Building # ########################### -# Define DISTNAME variable. -# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME -source contrib/gitian-descriptors/assign_DISTNAME - -GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then @@ -267,7 +285,7 @@ mkdir -p "$DISTSRC" sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool # Build Bitcoin Core - make --jobs="$MAX_JOBS" ${V:+V=1} + make --jobs="$JOBS" ${V:+V=1} # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} @@ -275,6 +293,7 @@ mkdir -p "$DISTSRC" # version symbols for Linux distro back-compatibility. make -C src --jobs=1 check-symbols ${V:+V=1} + mkdir -p ${OUTDIR} # Make the os-specific installers case "$HOST" in *mingw*) @@ -306,9 +325,6 @@ mkdir -p "$DISTSRC" osx_volname \ contrib/macdeploy/detached-sig-{apply,create}.sh \ "${BASEPREFIX}/${HOST}"/native/bin/dmg - for util in codesign_allocate pagestuff; do - cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util" - done mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" @@ -344,7 +360,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac @@ -394,21 +410,21 @@ mkdir -p "$DISTSRC" || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) ;; esac - ) -) + ) # $DISTSRC/installed -case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) - ) - ;; -esac + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..33a319012c --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=../../shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=../../shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invokable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix-<blah> + +EOF +exit 1 +fi + +################ +# Set common variables +################ + +VERSION="${VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 4228532cb1..910a9dd6f6 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -214,6 +214,7 @@ chain for " target " development.")) gzip xz zlib + (list zlib "static") ;; Build tools gnu-make libtool @@ -227,7 +228,8 @@ chain for " target " development.")) ;; Git git ;; Native gcc 7 toolchain - gcc-toolchain-7) + gcc-toolchain-7 + (list gcc-toolchain-7 "static")) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows @@ -237,5 +239,5 @@ chain for " target " development.")) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) + (list clang-toolchain-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) (else '()))))) diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 8408545a21..1cde19efd1 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -16,7 +16,7 @@ LOCALNET_V4="192.168.0.0/16" #defines the IPv6 address space for which you wish to disable rate limiting LOCALNET_V6="fe80::/10" -#delete existing rules +#delete existing rules ('Error: Cannot delete qdisc with handle of zero.' means there weren't any.) tc qdisc del dev ${IF} root #add root class diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..37bac1f38d --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} diff --git a/depends/Makefile b/depends/Makefile index 4cd4d72fc2..dadb21515a 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -141,8 +141,8 @@ build_id_string+=system_clang $(host_arch)_$(host_os)_id_string+=system_clang endif -build_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) -$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) +build_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) +$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) qrencode_packages_$(NO_QR) = $(qrencode_packages) diff --git a/depends/funcs.mk b/depends/funcs.mk index c0159f0e38..6e1a922f89 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -140,7 +140,13 @@ $(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) -$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" + +# Setting a --build type that differs from --host will explicitly enable +# cross-compilation mode. Note that --build defaults to the output of +# config.guess, which is what we set it too here. This also quells autoconf +# warnings, "If you wanted to set the --build type, don't use --host.", +# when using versions older than 2.70. +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 646e97837a..dd71697f0f 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -12,14 +12,13 @@ ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) # FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang # from llvm.org -# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG -# is empty +# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty darwin_native_toolchain=native_cctools clang_prog=$(build_prefix)/bin/clang clangxx_prog=$(clang_prog)++ -clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version) +clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) else # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's # system diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index d56b636695..c789dab74c 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -5,81 +5,19 @@ $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875 $(package)_build_subdir=cctools $(package)_patches=ld64_disable_threading.patch - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -$(package)_clang_version=8.0.0 -$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) -$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 -endif - -$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef -$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive -$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz -$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz -$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 - -$(package)_extra_sources=$($(package)_libtapi_file_name) -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -$(package)_extra_sources += $($(package)_clang_file_name) -endif - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) -endef -else -define $(package)_fetch_cmds -$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) -endef -endif - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -define $(package)_extract_cmds - mkdir -p $($(package)_extract_dir) && \ - echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \ - mkdir -p libtapi && \ - tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ - tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ - rm -f toolchain/lib/libc++abi.so* && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source) -endef -else -define $(package)_extract_cmds - mkdir -p $($(package)_extract_dir) && \ - echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir -p libtapi && \ - tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source) -endef -endif +$(package)_dependencies=native_libtapi define $(package)_set_vars - $(package)_config_opts=--target=$(host) --with-libtapi=$($(package)_extract_dir) + $(package)_config_opts=--target=$(host) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) - $(package)_config_opts+=--enable-lto-support --with-llvm-config=$($(package)_extract_dir)/toolchain/bin/llvm-config - $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang - $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ - else - $(package)_cc=clang - $(package)_cxx=clang++ + $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config endif + $(package)_cc=$(clang_prog) + $(package)_cxx=$(clangxx_prog) endef define $(package)_preprocess_cmds - CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ - CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch endef @@ -91,26 +29,10 @@ define $(package)_build_cmds $(MAKE) endef -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ - cd $($(package)_extract_dir) && \ - cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \ - cd $($(package)_extract_dir)/toolchain && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil + $(MAKE) DESTDIR=$($(package)_staging_dir) install endef -else -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ - cd $($(package)_extract_dir) && \ - cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ + +define $(package)_postprocess_cmds + rm -rf share endef -endif diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk new file mode 100644 index 0000000000..741ddacf09 --- /dev/null +++ b/depends/packages/native_clang.mk @@ -0,0 +1,26 @@ +package=native_clang +$(package)_version=8.0.0 +$(package)_download_path=https://releases.llvm.org/$($(package)_version) +$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_file_name=clang-llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz +$(package)_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 + +define $(package)_preprocess_cmds + rm -f $($(package)_extract_dir)/lib/libc++abi.so* +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ + mkdir -p $($(package)_staging_prefix_dir)/bin && \ + mkdir -p $($(package)_staging_prefix_dir)/include && \ + cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ + cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ + cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ + cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ + cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ +endef + +define $(package)_postprocess_cmds + rmdir include +endef diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index 035b767188..c7c8adef41 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -12,7 +12,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" .. + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. endef define $(package)_build_cmds diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk new file mode 100644 index 0000000000..d7ac4156a7 --- /dev/null +++ b/depends/packages/native_libtapi.mk @@ -0,0 +1,20 @@ +package=native_libtapi +$(package)_version=3efb201881e7a76a21e0554906cf306432539cef +$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive +$(package)_download_file=$($(package)_version).tar.gz +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +$(package)_dependencies=native_clang +endif + +define $(package)_build_cmds + CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh +endef + +define $(package)_stage_cmds + ./install.sh && \ + mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ + cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 810068b686..9094e327ef 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -24,5 +24,10 @@ darwin_native_packages = native_ds_store native_mac_alias $(host_arch)_$(host_os)_native_packages += native_b2 ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +darwin_native_packages+= native_clang +endif + endif diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index ee6796e2ad..ae83eae911 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,7 +10,7 @@ $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no $(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch $(package)_patches+= drop_lrelease_dependency.patch no_sdk_version_check.patch $(package)_patches+= fix_qpainter_non_determinism.patch fix_lib_paths.patch fix_android_pch.patch -$(package)_patches+= fix_bigsur_drawing.patch +$(package)_patches+= fix_bigsur_drawing.patch qtbase-moc-ignore-gcc-macro.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=e1de58ed108b7e0a138815ea60fd46a2c4e1fc31396a707e5630e92de79c53de @@ -232,6 +232,7 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_bigsur_drawing.patch && \ + patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch new file mode 100644 index 0000000000..0358bea6e9 --- /dev/null +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -0,0 +1,17 @@ +The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles +on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is +not supposed to be looking there to begin with. + +Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 + +diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp +--- a/qtbase/src/tools/moc/main.cpp ++++ b/qtbase/src/tools/moc/main.cpp +@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) + dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); + pp.macros["__attribute__"] = dummyVariadicFunctionMacro; + pp.macros["__declspec"] = dummyVariadicFunctionMacro; ++ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; + + QString filename; + QString output; diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 0a78cdff20..2130332eec 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -812,7 +812,7 @@ class ChainstateManager { public: ... - bool ProcessNewBlock(...) EXCLUSIVE_LOCKS_REQUIRED(!::cs_main); + bool ProcessNewBlock(...) LOCKS_EXCLUDED(::cs_main); ... } diff --git a/doc/release-notes-18077.md b/doc/release-notes-18077.md deleted file mode 100644 index 4034a4c19c..0000000000 --- a/doc/release-notes-18077.md +++ /dev/null @@ -1,10 +0,0 @@ -P2P and network changes ------------------------ - -- Added NAT-PMP port mapping support via [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html) - -Command-line options --------------------- - -- The `-natpmp` option has been added to use NAT-PMP to map the listening port. If both UPnP -and NAT-PMP are enabled, a successful allocation from UPnP prevails over one from NAT-PMP. diff --git a/doc/release-notes-18466.md b/doc/release-notes-18466.md deleted file mode 100644 index e46d5064a3..0000000000 --- a/doc/release-notes-18466.md +++ /dev/null @@ -1,10 +0,0 @@ -Low-level RPC changes ---------------------- - -- Error codes have been updated to be more accurate for the following error cases (#18466): - - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature - is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). diff --git a/doc/release-notes-19776.md b/doc/release-notes-19776.md deleted file mode 100644 index 5553c5a7bd..0000000000 --- a/doc/release-notes-19776.md +++ /dev/null @@ -1,9 +0,0 @@ -Updated RPCs ------------- - -- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and - `bip152_hb_from`, that respectively indicate whether we selected a peer to be - in compact blocks high-bandwidth mode or whether a peer selected us as a - compact blocks high-bandwidth peer. High-bandwidth peers send new block - announcements via a `cmpctblock` message rather than the usual inv/headers - announcements. See BIP 152 for more details. (#19776) diff --git a/doc/release-notes-20861.md b/doc/release-notes-20861.md deleted file mode 100644 index 5c68e4ab0c..0000000000 --- a/doc/release-notes-20861.md +++ /dev/null @@ -1,13 +0,0 @@ -Updated RPCs ------------- - -- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) - being implemented, behavior for all RPCs that accept addresses is changed when - a native witness version 1 (or higher) is passed. These now require a Bech32m - encoding instead of a Bech32 one, and Bech32m encoding will be used for such - addresses in RPC output as well. No version 1 addresses should be created - for mainnet until consensus rules are adopted that give them meaning - (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). - Once that happens, Bech32m is expected to be used for them, so this shouldn't - affect any production systems, but may be observed on other networks where such - addresses already have meaning (like signet). diff --git a/doc/release-notes.md b/doc/release-notes.md index e46380b38f..334dfa80a4 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -59,8 +59,30 @@ Notable changes P2P and network changes ----------------------- +- Added NAT-PMP port mapping support via + [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) + Updated RPCs ------------ + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). (#20861) + +- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and + `bip152_hb_from`, that respectively indicate whether we selected a peer to be + in compact blocks high-bandwidth mode or whether a peer selected us as a + compact blocks high-bandwidth peer. High-bandwidth peers send new block + announcements via a `cmpctblock` message rather than the usual inv/headers + announcements. See BIP 152 for more details. (#19776) + - `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, and `whitelisted`, which were previously deprecated in 0.21. Instead of `addnode`, the `connection_type` field returns manual. Instead of @@ -72,8 +94,8 @@ Updated RPCs `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no longer returned in the responses by default): `addresses`, `reqSigs`. The `-deprecatedrpc=addresses` flag must be passed for these fields to be - included in the RPC response. This flag/option will be available until v23, at which - point the deprecation will be removed entirely. Note that these fields are attributes of + included in the RPC response. This flag/option will be available only for this major release, after which + the deprecation will be removed entirely. Note that these fields are attributes of the `scriptPubKey` object returned in the RPC response. However, in the response of `decodescript` these fields are top-level attributes, and included again as attributes of the `scriptPubKey` object. (#20286) @@ -93,6 +115,10 @@ Build System New settings ------------ +- The `-natpmp` option has been added to use NAT-PMP to map the listening port. + If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP + prevails over one from NAT-PMP. (#18077) + Updated settings ---------------- @@ -110,6 +136,9 @@ Wallet The RPC returns public versions of all imported descriptors, including their timestamp and flags. For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) +- The `bumpfee` RPC is not available with wallets that have private keys + disabled. `psbtbumpfee` can be used instead. (#20891) + GUI changes ----------- @@ -118,11 +147,20 @@ Low-level changes RPC --- + - The RPC server can process a limited number of simultaneous RPC requests. - Previously, if this limit was exceeded, `bitcoind` would respond with + Previously, if this limit was exceeded, the RPC server would respond with [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) +- Error codes have been updated to be more accurate for the following error cases (#18466): + - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature + is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). + Tests ----- diff --git a/src/Makefile.am b/src/Makefile.am index 9903c2e9b3..4e09c86ebd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -243,7 +243,6 @@ BITCOIN_CORE_H = \ util/moneystr.h \ util/rbf.h \ util/readwritefile.h \ - util/ref.h \ util/settings.h \ util/sock.h \ util/spanparsing.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e30f3985b9..570f011f7a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -112,7 +112,6 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ - test/ref_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index f7f393ccac..85e50ebf70 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -26,6 +26,7 @@ libtest_util_a_SOURCES = \ test/util/logging.cpp \ test/util/mining.cpp \ test/util/net.cpp \ + test/util/script.cpp \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 1b4ca3e9a8..80ab69c131 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -16,7 +16,7 @@ #include <node/ui_interface.h> #include <noui.h> #include <shutdown.h> -#include <util/ref.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> @@ -24,6 +24,7 @@ #include <util/translation.h> #include <util/url.h> +#include <any> #include <functional> #include <optional> @@ -142,7 +143,7 @@ static bool AppInit(int argc, char* argv[]) // end, which is interpreted as failure to start. TokenPipeEnd daemon_ep; #endif - util::Ref context{node}; + std::any context{&node}; try { if (!CheckDataDirOption()) { diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 1bfd059c54..163d234f80 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -56,7 +56,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits } /** - * Main network + * Main network on which people trade goods and services. */ class CMainParams : public CChainParams { public: @@ -173,7 +173,7 @@ public: }; /** - * Testnet (v3) + * Testnet (v3): public test network which is reset from time to time. */ class CTestNetParams : public CChainParams { public: @@ -266,7 +266,7 @@ public: }; /** - * Signet + * Signet: test network with an additional consensus parameter (see BIP325). */ class SigNetParams : public CChainParams { public: @@ -373,7 +373,8 @@ public: }; /** - * Regression test + * Regression test: intended for private networks only. Has minimal difficulty to ensure that + * blocks can be found instantly. */ class CRegTestParams : public CChainParams { public: diff --git a/src/chainparams.h b/src/chainparams.h index 32b208e9f3..6f23199986 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -59,10 +59,7 @@ struct ChainTxData { /** * CChainParams defines various tweakable parameters of a given instance of the - * Bitcoin system. There are three: the main network on which people trade goods - * and services, the public test network which gets reset from time to time and - * a regression test mode which is intended for private networks only. It has - * minimal difficulty to ensure that blocks can be found instantly. + * Bitcoin system. */ class CChainParams { diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 867ddb090e..16ab38e0b2 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -144,7 +144,7 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return multiUserAuthorized(strUserPass); } -static bool HTTPReq_JSONRPC(const util::Ref& context, HTTPRequest* req) +static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) { // JSONRPC handles only POST if (req->GetRequestMethod() != HTTPRequest::POST) { @@ -288,7 +288,7 @@ static bool InitRPCAuthentication() return true; } -bool StartHTTPRPC(const util::Ref& context) +bool StartHTTPRPC(const std::any& context) { LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) diff --git a/src/httprpc.h b/src/httprpc.h index 97af6f7bb1..5a3b990646 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -5,14 +5,12 @@ #ifndef BITCOIN_HTTPRPC_H #define BITCOIN_HTTPRPC_H -namespace util { -class Ref; -} // namespace util +#include <any> /** Start HTTP RPC subsystem. * Precondition; HTTP and RPC has been started. */ -bool StartHTTPRPC(const util::Ref& context); +bool StartHTTPRPC(const std::any& context); /** Interrupt HTTP RPC subsystem. */ void InterruptHTTPRPC(); @@ -24,7 +22,7 @@ void StopHTTPRPC(); /** Start HTTP REST subsystem. * Precondition; HTTP and RPC has been started. */ -void StartREST(const util::Ref& context); +void StartREST(const std::any& context); /** Interrupt RPC REST subsystem. */ void InterruptREST(); diff --git a/src/index/base.h b/src/index/base.h index 8559e3cb64..ac3c429a5a 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -109,7 +109,7 @@ public: /// sync once and only needs to process blocks in the ValidationInterface /// queue. If the index is catching up from far behind, this method does /// not block and immediately returns false. - bool BlockUntilSyncedToCurrentChain() const; + bool BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main); void Interrupt(); diff --git a/src/init.cpp b/src/init.cpp index 8b1f531b81..17b216573f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -788,7 +788,7 @@ static bool InitSanityCheck() return true; } -static bool AppInitServers(const util::Ref& context, NodeContext& node) +static bool AppInitServers(const std::any& context, NodeContext& node) { const ArgsManager& args = *Assert(node.args); RPCServer::OnStarted(&OnRPCStarted); @@ -1277,7 +1277,7 @@ bool AppInitInterfaces(NodeContext& node) return true; } -bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) +bool AppInitMain(const std::any& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { const ArgsManager& args = *Assert(node.args); const CChainParams& chainparams = Params(); diff --git a/src/init.h b/src/init.h index 34bca09dd1..5d01d4c1ac 100644 --- a/src/init.h +++ b/src/init.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_INIT_H #define BITCOIN_INIT_H +#include <any> #include <memory> #include <string> @@ -22,9 +23,6 @@ struct BlockAndHeaderTipInfo; namespace boost { class thread_group; } // namespace boost -namespace util { -class Ref; -} // namespace util /** Interrupt threads */ void Interrupt(NodeContext& node); @@ -66,7 +64,7 @@ bool AppInitInterfaces(NodeContext& node); * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); +bool AppInitMain(const std::any& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); /** * Register all arguments with the ArgsManager diff --git a/src/miner.cpp b/src/miner.cpp index fe7a54c052..8a9406f810 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -55,9 +55,10 @@ BlockAssembler::Options::Options() { nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; } -BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options) +BlockAssembler::BlockAssembler(CChainState& chainstate, const CTxMemPool& mempool, const CChainParams& params, const Options& options) : chainparams(params), - m_mempool(mempool) + m_mempool(mempool), + m_chainstate(chainstate) { blockMinFeeRate = options.blockMinFeeRate; // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: @@ -79,8 +80,8 @@ static BlockAssembler::Options DefaultOptions() return options; } -BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params) - : BlockAssembler(mempool, params, DefaultOptions()) {} +BlockAssembler::BlockAssembler(CChainState& chainstate, const CTxMemPool& mempool, const CChainParams& params) + : BlockAssembler(chainstate, mempool, params, DefaultOptions()) {} void BlockAssembler::resetBlock() { @@ -96,7 +97,7 @@ void BlockAssembler::resetBlock() nFees = 0; } -std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(CChainState& chainstate, const CScript& scriptPubKeyIn) +std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) { int64_t nTimeStart = GetTimeMicros(); @@ -114,8 +115,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(CChainState& chai pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end LOCK2(cs_main, m_mempool.cs); - assert(std::addressof(*::ChainActive().Tip()) == std::addressof(*chainstate.m_chain.Tip())); - CBlockIndex* pindexPrev = chainstate.m_chain.Tip(); + assert(std::addressof(*::ChainActive().Tip()) == std::addressof(*m_chainstate.m_chain.Tip())); + CBlockIndex* pindexPrev = m_chainstate.m_chain.Tip(); assert(pindexPrev != nullptr); nHeight = pindexPrev->nHeight + 1; @@ -174,8 +175,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(CChainState& chai pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); BlockValidationState state; - assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); - if (!TestBlockValidity(state, chainparams, chainstate, *pblock, pindexPrev, false, false)) { + assert(std::addressof(::ChainstateActive()) == std::addressof(m_chainstate)); + if (!TestBlockValidity(state, chainparams, m_chainstate, *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } int64_t nTime2 = GetTimeMicros(); diff --git a/src/miner.h b/src/miner.h index 023635814c..c400c90f6c 100644 --- a/src/miner.h +++ b/src/miner.h @@ -146,6 +146,7 @@ private: int64_t nLockTimeCutoff; const CChainParams& chainparams; const CTxMemPool& m_mempool; + CChainState& m_chainstate; public: struct Options { @@ -154,11 +155,11 @@ public: CFeeRate blockMinFeeRate; }; - explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params); - explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options); + explicit BlockAssembler(CChainState& chainstate, const CTxMemPool& mempool, const CChainParams& params); + explicit BlockAssembler(CChainState& chainstate, const CTxMemPool& mempool, const CChainParams& params, const Options& options); /** Construct a new block template with coinbase to scriptPubKeyIn */ - std::unique_ptr<CBlockTemplate> CreateNewBlock(CChainState& chainstate, const CScript& scriptPubKeyIn); + std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); inline static std::optional<int64_t> m_last_block_num_txs{}; inline static std::optional<int64_t> m_last_block_weight{}; @@ -201,6 +202,7 @@ private: void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +// TODO just accept a CBlockIndex* /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ void RegenerateCommitments(CBlock& block, BlockManager& blockman); diff --git a/src/net.cpp b/src/net.cpp index e9170d06cd..ae38acdc3c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -31,6 +31,10 @@ #include <fcntl.h> #endif +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS +#include <ifaddrs.h> +#endif + #ifdef USE_POLL #include <poll.h> #endif @@ -1252,9 +1256,10 @@ void CConnman::NotifyNumConnectionsChanged() } } -bool CConnman::RunInactivityChecks(const CNode& node) const +bool CConnman::ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now_in) const { - return GetSystemTimeInSeconds() > node.nTimeConnected + m_peer_connect_timeout; + const int64_t now = now_in ? now_in.value() : GetSystemTimeInSeconds(); + return node.nTimeConnected + m_peer_connect_timeout < now; } bool CConnman::InactivityCheck(const CNode& node) const @@ -1263,6 +1268,8 @@ bool CConnman::InactivityCheck(const CNode& node) const // use setmocktime in the tests). int64_t now = GetSystemTimeInSeconds(); + if (!ShouldRunInactivityChecks(node, now)) return false; + if (node.nLastRecv == 0 || node.nLastSend == 0) { LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); return true; @@ -1559,7 +1566,7 @@ void CConnman::SocketHandler() if (bytes_sent) RecordBytesSent(bytes_sent); } - if (RunInactivityChecks(*pnode) && InactivityCheck(*pnode)) pnode->fDisconnect = true; + if (InactivityCheck(*pnode)) pnode->fDisconnect = true; } { LOCK(cs_vNodes); @@ -549,8 +549,9 @@ public: std::vector<CAddress> vAddrToSend; std::unique_ptr<CRollingBloomFilter> m_addr_known{nullptr}; bool fGetAddr{false}; - std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; - std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; + Mutex m_addr_send_times_mutex; + std::chrono::microseconds m_next_addr_send GUARDED_BY(m_addr_send_times_mutex){0}; + std::chrono::microseconds m_next_local_addr_send GUARDED_BY(m_addr_send_times_mutex){0}; struct TxRelay { mutable RecursiveMutex cs_filter; @@ -1017,8 +1018,8 @@ public: void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } - /** Return true if the peer has been connected for long enough to do inactivity checks. */ - bool RunInactivityChecks(const CNode& node) const; + /** Return true if we should disconnect the peer for failing an inactivity check. */ + bool ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now=std::nullopt) const; private: struct ListenSocket { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index e4054968c0..d98d47fd2a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -33,6 +33,7 @@ #include <util/system.h> #include <validation.h> +#include <algorithm> #include <memory> #include <optional> #include <typeinfo> @@ -317,8 +318,13 @@ private: void PushNodeVersion(CNode& pnode, int64_t nTime); /** Send a ping message every PING_INTERVAL or if requested via RPC. May - * mark the peer to be disconnected if a ping has timed out. */ - void MaybeSendPing(CNode& node_to, Peer& peer); + * mark the peer to be disconnected if a ping has timed out. + * We use mockable time for ping timeouts, so setmocktime may cause pings + * to time out. */ + void MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::microseconds now); + + /** Send `addr` messages on a regular schedule. */ + void MaybeSendAddr(CNode& node, std::chrono::microseconds current_time); const CChainParams& m_chainparams; CConnman& m_connman; @@ -442,7 +448,7 @@ private: /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ CTransactionRef FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main); - void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex); + void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(peer.m_getdata_requests_mutex) LOCKS_EXCLUDED(::cs_main); /** Relay map (txid or wtxid -> CTransactionRef) */ typedef std::map<uint256, CTransactionRef> MapRelay; @@ -476,19 +482,70 @@ private: /** Offset into vExtraTxnForCompact to insert the next tx */ size_t vExtraTxnForCompactIt GUARDED_BY(g_cs_orphans) = 0; + /** Check whether the last unknown block a peer advertised is not yet known. */ void ProcessBlockAvailability(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Update tracking information about which blocks a peer is assumed to have. */ void UpdateBlockAvailability(NodeId nodeid, const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool CanDirectFetch() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * To prevent fingerprinting attacks, only send blocks/headers outside of + * the active chain if they are no more than a month older (both in time, + * and in best equivalent proof of work) than the best header chain we know + * about and we fully-validated them at some point. + */ bool BlockRequestAllowed(const CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); void ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& inv); + + /** + * Validation logic for compact filters request handling. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] peer The peer that we received the request from + * @param[in] filter_type The filter type the request is for. Must be basic filters. + * @param[in] start_height The start height for the request + * @param[in] stop_hash The stop_hash for the request + * @param[in] max_height_diff The maximum number of items permitted to request, as specified in BIP 157 + * @param[out] stop_index The CBlockIndex for the stop_hash block, if the request can be serviced. + * @param[out] filter_index The filter index, if the request can be serviced. + * @return True if the request can be serviced. + */ bool PrepareBlockFilterRequest(CNode& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, BlockFilterIndex*& filter_index); + + /** + * Handle a cfilters request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] peer The peer that we received the request from + * @param[in] vRecv The raw message received + */ void ProcessGetCFilters(CNode& peer, CDataStream& vRecv); + + /** + * Handle a cfheaders request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] peer The peer that we received the request from + * @param[in] vRecv The raw message received + */ void ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv); + + /** + * Handle a getcfcheckpt request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] peer The peer that we received the request from + * @param[in] vRecv The raw message received + */ void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv); }; } // namespace @@ -742,7 +799,6 @@ static bool PeerHasHeader(CNodeState *state, const CBlockIndex *pindex) EXCLUSIV return false; } -/** Check whether the last unknown block a peer advertised is not yet known. */ void PeerManagerImpl::ProcessBlockAvailability(NodeId nodeid) { CNodeState *state = State(nodeid); assert(state != nullptr); @@ -758,7 +814,6 @@ void PeerManagerImpl::ProcessBlockAvailability(NodeId nodeid) { } } -/** Update tracking information about which blocks a peer is assumed to have. */ void PeerManagerImpl::UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { CNodeState *state = State(nodeid); assert(state != nullptr); @@ -1185,16 +1240,6 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat return false; } - -////////////////////////////////////////////////////////////////////////////// -// -// blockchain -> download logic notification -// - -// To prevent fingerprinting attacks, only send blocks/headers outside of the -// active chain if they are no more than a month older (both in time, and in -// best equivalent proof of work) than the best header chain we know about and -// we fully-validated them at some point. bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) { AssertLockHeld(cs_main); @@ -2095,20 +2140,6 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) m_mempool.check(m_chainman.ActiveChainstate()); } -/** - * Validation logic for compact filters request handling. - * - * May disconnect from the peer in the case of a bad request. - * - * @param[in] peer The peer that we received the request from - * @param[in] filter_type The filter type the request is for. Must be basic filters. - * @param[in] start_height The start height for the request - * @param[in] stop_hash The stop_hash for the request - * @param[in] max_height_diff The maximum number of items permitted to request, as specified in BIP 157 - * @param[out] stop_index The CBlockIndex for the stop_hash block, if the request can be serviced. - * @param[out] filter_index The filter index, if the request can be serviced. - * @return True if the request can be serviced. - */ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, @@ -2162,14 +2193,6 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, return true; } -/** - * Handle a cfilters request. - * - * May disconnect from the peer in the case of a bad request. - * - * @param[in] peer The peer that we received the request from - * @param[in] vRecv The raw message received - */ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) { uint8_t filter_type_ser; @@ -2201,14 +2224,6 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) } } -/** - * Handle a cfheaders request. - * - * May disconnect from the peer in the case of a bad request. - * - * @param[in] peer The peer that we received the request from - * @param[in] vRecv The raw message received - */ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) { uint8_t filter_type_ser; @@ -2253,14 +2268,6 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) m_connman.PushMessage(&peer, std::move(msg)); } -/** - * Handle a getcfcheckpt request. - * - * May disconnect from the peer in the case of a bad request. - * - * @param[in] peer The peer that we received the request from - * @param[in] vRecv The raw message received - */ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) { uint8_t filter_type_ser; @@ -4100,13 +4107,9 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() } } -void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer) +void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::microseconds now) { - // Use mockable time for ping timeouts. - // This means that setmocktime may cause pings to time out. - auto now = GetTime<std::chrono::microseconds>(); - - if (m_connman.RunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + if (m_connman.ShouldRunInactivityChecks(node_to) && peer.m_ping_nonce_sent && now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); node_to.fDisconnect = true; @@ -4144,6 +4147,75 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer) } } +void PeerManagerImpl::MaybeSendAddr(CNode& node, std::chrono::microseconds current_time) +{ + // Nothing to do for non-address-relay peers + if (!node.RelayAddrsWithConn()) return; + + assert(node.m_addr_known); + + LOCK(node.m_addr_send_times_mutex); + // Periodically advertise our local address to the peer. + if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload() && + node.m_next_local_addr_send < current_time) { + // If we've sent before, clear the bloom filter for the peer, so that our + // self-announcement will actually go out. + // This might be unnecessary if the bloom filter has already rolled + // over since our last self-announcement, but there is only a small + // bandwidth cost that we can incur by doing this (which happens + // once a day on average). + if (node.m_next_local_addr_send != 0us) { + node.m_addr_known->reset(); + } + if (std::optional<CAddress> local_addr = GetLocalAddrForPeer(&node)) { + FastRandomContext insecure_rand; + node.PushAddress(*local_addr, insecure_rand); + } + node.m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + } + + // We sent an `addr` message to this peer recently. Nothing more to do. + if (current_time <= node.m_next_addr_send) return; + + node.m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); + + if (!Assume(node.vAddrToSend.size() <= MAX_ADDR_TO_SEND)) { + // Should be impossible since we always check size before adding to + // vAddrToSend. Recover by trimming the vector. + node.vAddrToSend.resize(MAX_ADDR_TO_SEND); + } + + // Remove addr records that the peer already knows about, and add new + // addrs to the m_addr_known filter on the same pass. + auto addr_already_known = [&node](const CAddress& addr) { + bool ret = node.m_addr_known->contains(addr.GetKey()); + if (!ret) node.m_addr_known->insert(addr.GetKey()); + return ret; + }; + node.vAddrToSend.erase(std::remove_if(node.vAddrToSend.begin(), node.vAddrToSend.end(), addr_already_known), + node.vAddrToSend.end()); + + // No addr messages to send + if (node.vAddrToSend.empty()) return; + + const char* msg_type; + int make_flags; + if (node.m_wants_addrv2) { + msg_type = NetMsgType::ADDRV2; + make_flags = ADDRV2_FORMAT; + } else { + msg_type = NetMsgType::ADDR; + make_flags = 0; + } + m_connman.PushMessage(&node, CNetMsgMaker(node.GetCommonVersion()).Make(make_flags, msg_type, node.vAddrToSend)); + node.vAddrToSend.clear(); + + // we only send the big addr message once + if (node.vAddrToSend.capacity() > 40) { + node.vAddrToSend.shrink_to_fit(); + } +} + namespace { class CompareInvMempoolOrder { @@ -4182,79 +4254,20 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // If we get here, the outgoing message serialization version is set and can't change. const CNetMsgMaker msgMaker(pto->GetCommonVersion()); - MaybeSendPing(*pto, *peer); + const auto current_time = GetTime<std::chrono::microseconds>(); + + MaybeSendPing(*pto, *peer, current_time); // MaybeSendPing may have marked peer for disconnection if (pto->fDisconnect) return true; + MaybeSendAddr(*pto, current_time); + { LOCK(cs_main); CNodeState &state = *State(pto->GetId()); - // Address refresh broadcast - auto current_time = GetTime<std::chrono::microseconds>(); - - if (fListen && pto->RelayAddrsWithConn() && - !m_chainman.ActiveChainstate().IsInitialBlockDownload() && - pto->m_next_local_addr_send < current_time) { - // If we've sent before, clear the bloom filter for the peer, so that our - // self-announcement will actually go out. - // This might be unnecessary if the bloom filter has already rolled - // over since our last self-announcement, but there is only a small - // bandwidth cost that we can incur by doing this (which happens - // once a day on average). - if (pto->m_next_local_addr_send != 0us) { - pto->m_addr_known->reset(); - } - if (std::optional<CAddress> local_addr = GetLocalAddrForPeer(pto)) { - FastRandomContext insecure_rand; - pto->PushAddress(*local_addr, insecure_rand); - } - pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); - } - - // - // Message: addr - // - if (pto->RelayAddrsWithConn() && pto->m_next_addr_send < current_time) { - pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); - std::vector<CAddress> vAddr; - vAddr.reserve(pto->vAddrToSend.size()); - assert(pto->m_addr_known); - - const char* msg_type; - int make_flags; - if (pto->m_wants_addrv2) { - msg_type = NetMsgType::ADDRV2; - make_flags = ADDRV2_FORMAT; - } else { - msg_type = NetMsgType::ADDR; - make_flags = 0; - } - - for (const CAddress& addr : pto->vAddrToSend) - { - if (!pto->m_addr_known->contains(addr.GetKey())) - { - pto->m_addr_known->insert(addr.GetKey()); - vAddr.push_back(addr); - // receiver rejects addr messages larger than MAX_ADDR_TO_SEND - if (vAddr.size() >= MAX_ADDR_TO_SEND) - { - m_connman.PushMessage(pto, msgMaker.Make(make_flags, msg_type, vAddr)); - vAddr.clear(); - } - } - } - pto->vAddrToSend.clear(); - if (!vAddr.empty()) - m_connman.PushMessage(pto, msgMaker.Make(make_flags, msg_type, vAddr)); - // we only send the big addr message once - if (pto->vAddrToSend.capacity() > 40) - pto->vAddrToSend.shrink_to_fit(); - } - // Start block sync if (pindexBestHeader == nullptr) pindexBestHeader = m_chainman.ActiveChain().Tip(); @@ -4489,7 +4502,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.clear(); } } - pto->m_tx_relay->m_last_mempool_req = GetTime<std::chrono::seconds>(); + pto->m_tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); } // Determine transactions to relay @@ -4577,7 +4590,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - current_time = GetTime<std::chrono::microseconds>(); if (state.m_stalling_since.count() && state.m_stalling_since < current_time - BLOCK_STALLING_TIMEOUT) { // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection diff --git a/src/node/coin.cpp b/src/node/coin.cpp index 263dcff657..23d4fa2aae 100644 --- a/src/node/coin.cpp +++ b/src/node/coin.cpp @@ -11,6 +11,7 @@ void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins) { assert(node.mempool); + assert(node.chainman); LOCK2(cs_main, node.mempool->cs); assert(std::addressof(::ChainstateActive()) == std::addressof(node.chainman->ActiveChainstate())); CCoinsViewCache& chain_view = node.chainman->ActiveChainstate().CoinsTip(); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 83f228aa7e..975651dcc4 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -8,11 +8,11 @@ #include <amount.h> #include <uint256.h> -#include <validation.h> #include <cstdint> #include <functional> +class BlockManager; class CCoinsView; enum class CoinStatsHashType { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 50c8c29175..7ad02f81dc 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -38,7 +38,6 @@ #include <uint256.h> #include <univalue.h> #include <util/check.h> -#include <util/ref.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> @@ -49,6 +48,7 @@ #include <config/bitcoin-config.h> #endif +#include <any> #include <memory> #include <optional> #include <utility> @@ -65,6 +65,8 @@ namespace node { namespace { class NodeImpl : public Node { +private: + ChainstateManager& chainman() { return *Assert(m_context->chainman); } public: explicit NodeImpl(NodeContext* context) { setContext(context); } void initLogging() override { InitLogging(*Assert(m_context->args)); } @@ -183,21 +185,28 @@ public: int getNumBlocks() override { LOCK(::cs_main); - assert(std::addressof(::ChainActive()) == std::addressof(m_context->chainman->ActiveChain())); - return m_context->chainman->ActiveChain().Height(); + assert(std::addressof(::ChainActive()) == std::addressof(chainman().ActiveChain())); + return chainman().ActiveChain().Height(); } uint256 getBestBlockHash() override { - assert(std::addressof(::ChainActive()) == std::addressof(m_context->chainman->ActiveChain())); - const CBlockIndex* tip = WITH_LOCK(::cs_main, return m_context->chainman->ActiveChain().Tip()); + const CBlockIndex* tip; + { + // TODO: Temporary scope to check correctness of refactored code. + // Should be removed manually after merge of + // https://github.com/bitcoin/bitcoin/pull/20158 + LOCK(cs_main); + assert(std::addressof(::ChainActive()) == std::addressof(chainman().ActiveChain())); + tip = chainman().ActiveChain().Tip(); + } return tip ? tip->GetBlockHash() : Params().GenesisBlock().GetHash(); } int64_t getLastBlockTime() override { LOCK(::cs_main); - assert(std::addressof(::ChainActive()) == std::addressof(m_context->chainman->ActiveChain())); - if (m_context->chainman->ActiveChain().Tip()) { - return m_context->chainman->ActiveChain().Tip()->GetBlockTime(); + assert(std::addressof(::ChainActive()) == std::addressof(chainman().ActiveChain())); + if (chainman().ActiveChain().Tip()) { + return chainman().ActiveChain().Tip()->GetBlockTime(); } return Params().GenesisBlock().GetBlockTime(); // Genesis block's time of current network } @@ -206,14 +215,22 @@ public: const CBlockIndex* tip; { LOCK(::cs_main); - assert(std::addressof(::ChainActive()) == std::addressof(m_context->chainman->ActiveChain())); - tip = m_context->chainman->ActiveChain().Tip(); + assert(std::addressof(::ChainActive()) == std::addressof(chainman().ActiveChain())); + tip = chainman().ActiveChain().Tip(); } return GuessVerificationProgress(Params().TxData(), tip); } bool isInitialBlockDownload() override { - assert(std::addressof(::ChainstateActive()) == std::addressof(m_context->chainman->ActiveChainstate())); - return m_context->chainman->ActiveChainstate().IsInitialBlockDownload(); + const CChainState* active_chainstate; + { + // TODO: Temporary scope to check correctness of refactored code. + // Should be removed manually after merge of + // https://github.com/bitcoin/bitcoin/pull/20158 + LOCK(::cs_main); + active_chainstate = &m_context->chainman->ActiveChainstate(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*active_chainstate)); + } + return active_chainstate->IsInitialBlockDownload(); } bool getReindex() override { return ::fReindex; } bool getImporting() override { return ::fImporting; } @@ -239,8 +256,8 @@ public: bool getUnspentOutput(const COutPoint& output, Coin& coin) override { LOCK(::cs_main); - assert(std::addressof(::ChainstateActive()) == std::addressof(m_context->chainman->ActiveChainstate())); - return m_context->chainman->ActiveChainstate().CoinsTip().GetCoin(output, coin); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainman().ActiveChainstate())); + return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); } WalletClient& walletClient() override { @@ -298,13 +315,13 @@ public: { m_context = context; if (context) { - m_context_ref.Set(*context); + m_context_ref = context; } else { - m_context_ref.Clear(); + m_context_ref.reset(); } } NodeContext* m_context{nullptr}; - util::Ref m_context_ref; + std::any m_context_ref; }; bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active) @@ -414,6 +431,8 @@ public: class ChainImpl : public Chain { +private: + ChainstateManager& chainman() { return *Assert(m_node.chainman); } public: explicit ChainImpl(NodeContext& node) : m_node(node) {} std::optional<int> getHeight() override @@ -450,8 +469,8 @@ public: bool checkFinalTx(const CTransaction& tx) override { LOCK(cs_main); - assert(std::addressof(::ChainActive()) == std::addressof(m_node.chainman->ActiveChain())); - return CheckFinalTx(m_node.chainman->ActiveChain().Tip(), tx); + assert(std::addressof(::ChainActive()) == std::addressof(chainman().ActiveChain())); + return CheckFinalTx(chainman().ActiveChain().Tip(), tx); } std::optional<int> findLocatorFork(const CBlockLocator& locator) override { @@ -516,8 +535,8 @@ public: double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); - assert(std::addressof(g_chainman) == std::addressof(*m_node.chainman)); - return GuessVerificationProgress(Params().TxData(), m_node.chainman->m_blockman.LookupBlockIndex(block_hash)); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(chainman().m_blockman)); + return GuessVerificationProgress(Params().TxData(), chainman().m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, std::optional<int> max_height) override { @@ -529,8 +548,8 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - assert(std::addressof(g_chainman) == std::addressof(*m_node.chainman)); - if (CBlockIndex* block = m_node.chainman->m_blockman.LookupBlockIndex(block_hash)) { + assert(std::addressof(g_chainman.m_blockman) == std::addressof(chainman().m_blockman)); + if (CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low @@ -621,8 +640,16 @@ public: } bool isReadyToBroadcast() override { return !::fImporting && !::fReindex && !isInitialBlockDownload(); } bool isInitialBlockDownload() override { - assert(std::addressof(::ChainstateActive()) == std::addressof(m_node.chainman->ActiveChainstate())); - return m_node.chainman->ActiveChainstate().IsInitialBlockDownload(); + const CChainState* active_chainstate; + { + // TODO: Temporary scope to check correctness of refactored code. + // Should be removed manually after merge of + // https://github.com/bitcoin/bitcoin/pull/20158 + LOCK(::cs_main); + active_chainstate = &chainman().ActiveChainstate(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*active_chainstate)); + } + return active_chainstate->IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } int64_t getAdjustedTime() override { return GetAdjustedTime(); } diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index f47e85aceb..691b2791d7 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -38,6 +38,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t bool callback_set = false; { // cs_main scope + assert(node.chainman); LOCK(cs_main); assert(std::addressof(::ChainstateActive()) == std::addressof(node.chainman->ActiveChainstate())); // If the transaction is already confirmed in the chain, don't do anything diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 0d9928d363..9e6e98b274 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -11,8 +11,10 @@ #include <univalue.h> #include <util/system.h> -#include <QDir> -#include <QtGlobal> +#include <QTest> + +#include <string> +#include <stdexcept> static RPCHelpMan rpcNestedTest_rpc() { @@ -24,9 +26,9 @@ static RPCHelpMan rpcNestedTest_rpc() {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, }, - {}, + RPCResult{RPCResult::Type::ANY, "", ""}, RPCExamples{""}, - [](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { return request.params.write(0, 0); }, }; @@ -68,13 +70,13 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo "); //whitespace at the end will be tolerated QVERIFY(result.substr(0,1) == "{"); - (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()[\"chain\"]")); //Quote path identifier are allowed, but look after a child containing the quotes in the key + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()[\"chain\"]"); //Quote path identifier are allowed, but look after a child containing the quotes in the key QVERIFY(result == "null"); - (RPCConsole::RPCExecuteCommandLine(m_node, result, "createrawtransaction [] {} 0")); //parameter not in brackets are allowed - (RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction([],{},0)")); //parameter in brackets are allowed + RPCConsole::RPCExecuteCommandLine(m_node, result, "createrawtransaction [] {} 0"); //parameter not in brackets are allowed + RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction([],{},0)"); //parameter in brackets are allowed QVERIFY(result == result2); - (RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction( [], {} , 0 )")); //whitespace between parameters is allowed + RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction( [], {} , 0 )"); //whitespace between parameters is allowed QVERIFY(result == result2); RPCConsole::RPCExecuteCommandLine(m_node, result, "getblock(getbestblockhash())[tx][0]", &filtered); @@ -123,11 +125,10 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); - // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax - (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo(")); //tolerate non closing brackets if we have no arguments - (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()()()")); //tolerate non command brackets + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo("); //tolerate non closing brackets if we have no arguments + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()()()"); //tolerate non command brackets QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo(True)"), UniValue); //invalid argument QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "a(getblockchaininfo(True))"), UniValue); //method not found QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tolerate empty arguments when using , diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 9248db1539..fa2a3a0607 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -38,7 +38,7 @@ #include <sys/utsname.h> #include <unistd.h> #endif -#if HAVE_DECL_GETIFADDRS +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS #include <ifaddrs.h> #endif #if HAVE_SYSCTL @@ -361,7 +361,7 @@ void RandAddStaticEnv(CSHA512& hasher) hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); } -#if HAVE_DECL_GETIFADDRS +#if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces struct ifaddrs *ifad = NULL; getifaddrs(&ifad); diff --git a/src/rest.cpp b/src/rest.cpp index 71426a4dc4..aa97470ca7 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -18,10 +18,12 @@ #include <sync.h> #include <txmempool.h> #include <util/check.h> -#include <util/ref.h> +#include <util/system.h> #include <validation.h> #include <version.h> +#include <any> + #include <boost/algorithm/string.hpp> #include <univalue.h> @@ -73,10 +75,10 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me * context is not found. * @returns Pointer to the node context or nullptr if not found. */ -static NodeContext* GetNodeContext(const util::Ref& context, HTTPRequest* req) +static NodeContext* GetNodeContext(const std::any& context, HTTPRequest* req) { - NodeContext* node = context.Has<NodeContext>() ? &context.Get<NodeContext>() : nullptr; - if (!node) { + auto node_context = util::AnyPtr<NodeContext>(context); + if (!node_context) { RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("%s:%d (%s)\n" "Internal bug detected: Node context not found!\n" @@ -84,7 +86,7 @@ static NodeContext* GetNodeContext(const util::Ref& context, HTTPRequest* req) __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)); return nullptr; } - return node; + return node_context; } /** @@ -94,14 +96,14 @@ static NodeContext* GetNodeContext(const util::Ref& context, HTTPRequest* req) * context mempool is not found. * @returns Pointer to the mempool or nullptr if no mempool found. */ -static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) +static CTxMemPool* GetMemPool(const std::any& context, HTTPRequest* req) { - NodeContext* node = context.Has<NodeContext>() ? &context.Get<NodeContext>() : nullptr; - if (!node || !node->mempool) { + auto node_context = util::AnyPtr<NodeContext>(context); + if (!node_context || !node_context->mempool) { RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); return nullptr; } - return node->mempool.get(); + return node_context->mempool.get(); } static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) @@ -151,7 +153,7 @@ static bool CheckWarmup(HTTPRequest* req) return true; } -static bool rest_headers(const util::Ref& context, +static bool rest_headers(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { @@ -293,12 +295,12 @@ static bool rest_block(HTTPRequest* req, } } -static bool rest_block_extended(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_extended(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, false); } @@ -306,7 +308,7 @@ static bool rest_block_notxdetails(const util::Ref& context, HTTPRequest* req, c // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp RPCHelpMan getblockchaininfo(); -static bool rest_chaininfo(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -329,7 +331,7 @@ static bool rest_chaininfo(const util::Ref& context, HTTPRequest* req, const std } } -static bool rest_mempool_info(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_info(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -353,7 +355,7 @@ static bool rest_mempool_info(const util::Ref& context, HTTPRequest* req, const } } -static bool rest_mempool_contents(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_contents(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; const CTxMemPool* mempool = GetMemPool(context, req); @@ -376,7 +378,7 @@ static bool rest_mempool_contents(const util::Ref& context, HTTPRequest* req, co } } -static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -435,7 +437,7 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin } } -static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) +static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -621,7 +623,7 @@ static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std: } } -static bool rest_blockhash_by_height(const util::Ref& context, HTTPRequest* req, +static bool rest_blockhash_by_height(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) return false; @@ -669,7 +671,7 @@ static bool rest_blockhash_by_height(const util::Ref& context, HTTPRequest* req, static const struct { const char* prefix; - bool (*handler)(const util::Ref& context, HTTPRequest* req, const std::string& strReq); + bool (*handler)(const std::any& context, HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -682,7 +684,7 @@ static const struct { {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; -void StartREST(const util::Ref& context) +void StartREST(const std::any& context) { for (const auto& up : uri_prefixes) { auto handler = [&context, up](HTTPRequest* req, const std::string& prefix) { return up.handler(context, req, prefix); }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f0ad141fa9..e1501d7254 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -30,7 +30,6 @@ #include <txdb.h> #include <txmempool.h> #include <undo.h> -#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> @@ -56,15 +55,16 @@ static Mutex cs_blockchange; static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); -NodeContext& EnsureNodeContext(const util::Ref& context) +NodeContext& EnsureNodeContext(const std::any& context) { - if (!context.Has<NodeContext>()) { + auto node_context = util::AnyPtr<NodeContext>(context); + if (!node_context) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Node context not found"); } - return context.Get<NodeContext>(); + return *node_context; } -CTxMemPool& EnsureMemPool(const util::Ref& context) +CTxMemPool& EnsureMemPool(const std::any& context) { const NodeContext& node = EnsureNodeContext(context); if (!node.mempool) { @@ -73,7 +73,7 @@ CTxMemPool& EnsureMemPool(const util::Ref& context) return *node.mempool; } -ChainstateManager& EnsureChainman(const util::Ref& context) +ChainstateManager& EnsureChainman(const std::any& context) { const NodeContext& node = EnsureNodeContext(context); if (!node.chainman) { @@ -82,7 +82,7 @@ ChainstateManager& EnsureChainman(const util::Ref& context) return *node.chainman; } -CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context) +CBlockPolicyEstimator& EnsureFeeEstimator(const std::any& context) { NodeContext& node = EnsureNodeContext(context); if (!node.fee_estimator) { @@ -1669,9 +1669,9 @@ static RPCHelpMan getchaintxstats() {RPCResult::Type::STR_HEX, "window_final_block_hash", "The hash of the final block in the window"}, {RPCResult::Type::NUM, "window_final_block_height", "The height of the final block in the window."}, {RPCResult::Type::NUM, "window_block_count", "Size of the window in number of blocks"}, - {RPCResult::Type::NUM, "window_tx_count", "The number of transactions in the window. Only returned if \"window_block_count\" is > 0"}, - {RPCResult::Type::NUM, "window_interval", "The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0"}, - {RPCResult::Type::NUM, "txrate", "The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0"}, + {RPCResult::Type::NUM, "window_tx_count", /* optional */ true, "The number of transactions in the window. Only returned if \"window_block_count\" is > 0"}, + {RPCResult::Type::NUM, "window_interval", /* optional */ true, "The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0"}, + {RPCResult::Type::NUM, "txrate", /* optional */ true, "The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0"}, }}, RPCExamples{ HelpExampleCli("getchaintxstats", "") diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index e719dfc702..cd04c9a10f 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -10,6 +10,7 @@ #include <streams.h> #include <sync.h> +#include <any> #include <stdint.h> #include <vector> @@ -23,9 +24,6 @@ class CTxMemPool; class ChainstateManager; class UniValue; struct NodeContext; -namespace util { -class Ref; -} // namespace util static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -58,10 +56,10 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); -NodeContext& EnsureNodeContext(const util::Ref& context); -CTxMemPool& EnsureMemPool(const util::Ref& context); -ChainstateManager& EnsureChainman(const util::Ref& context); -CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context); +NodeContext& EnsureNodeContext(const std::any& context); +CTxMemPool& EnsureMemPool(const std::any& context); +ChainstateManager& EnsureChainman(const std::any& context); +CBlockPolicyEstimator& EnsureFeeEstimator(const std::any& context); /** * Helper to create UTXO snapshots given a chainstate and a file handle. diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index fd780ba782..72ad0df199 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -150,7 +150,7 @@ static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& me UniValue blockHashes(UniValue::VARR); while (nHeight < nHeightEnd && !ShutdownRequested()) { - std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(mempool, Params()).CreateNewBlock(::ChainstateActive(), coinbase_script)); + std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(chainman.ActiveChainstate(), mempool, Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; @@ -358,7 +358,7 @@ static RPCHelpMan generateblock() LOCK(cs_main); CTxMemPool empty_mempool; - std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(empty_mempool, chainparams).CreateNewBlock(::ChainstateActive(), coinbase_script)); + std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(::ChainstateActive(), empty_mempool, chainparams).CreateNewBlock(coinbase_script)); if (!blocktemplate) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); } @@ -748,7 +748,7 @@ static RPCHelpMan getblocktemplate() // Create new block CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = BlockAssembler(mempool, Params()).CreateNewBlock(::ChainstateActive(), scriptDummy); + pblocktemplate = BlockAssembler(::ChainstateActive(), mempool, Params()).CreateNewBlock(scriptDummy); if (!pblocktemplate) throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 143be1274e..8e98919391 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -17,7 +17,6 @@ #include <script/descriptor.h> #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() -#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> @@ -391,8 +390,9 @@ static RPCHelpMan setmocktime() throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime can not be negative: %s.", time)); } SetMockTime(time); - if (request.context.Has<NodeContext>()) { - for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { + auto node_context = util::AnyPtr<NodeContext>(request.context); + if (node_context) { + for (const auto& chain_client : node_context->chain_clients) { chain_client->setMockTime(time); } } @@ -424,11 +424,11 @@ static RPCHelpMan mockscheduler() throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); } + auto node_context = util::AnyPtr<NodeContext>(request.context); // protect against null pointer dereference - CHECK_NONFATAL(request.context.Has<NodeContext>()); - NodeContext& node = request.context.Get<NodeContext>(); - CHECK_NONFATAL(node.scheduler); - node.scheduler->MockForward(std::chrono::seconds(delta_seconds)); + CHECK_NONFATAL(node_context); + CHECK_NONFATAL(node_context->scheduler); + node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); return NullUniValue; }, @@ -628,7 +628,7 @@ static RPCHelpMan echo(const std::string& name) {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, }, - RPCResult{RPCResult::Type::NONE, "", "Returns whatever was passed in"}, + RPCResult{RPCResult::Type::ANY, "", "Returns whatever was passed in"}, RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { diff --git a/src/rpc/request.h b/src/rpc/request.h index 27d06f3c92..e1569673f6 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -6,14 +6,11 @@ #ifndef BITCOIN_RPC_REQUEST_H #define BITCOIN_RPC_REQUEST_H +#include <any> #include <string> #include <univalue.h> -namespace util { -class Ref; -} // namespace util - UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); @@ -38,14 +35,14 @@ public: std::string URI; std::string authUser; std::string peerAddr; - const util::Ref& context; + const std::any& context; - explicit JSONRPCRequest(const util::Ref& context) : id(NullUniValue), params(NullUniValue), context(context) {} + explicit JSONRPCRequest(const std::any& context) : id(NullUniValue), params(NullUniValue), context(context) {} //! Initializes request information from another request object and the //! given context. The implementation should be updated if any members are //! added or removed above. - JSONRPCRequest(const JSONRPCRequest& other, const util::Ref& context) + JSONRPCRequest(const JSONRPCRequest& other, const std::any& context) : id(other.id), strMethod(other.strMethod), params(other.params), mode(other.mode), URI(other.URI), authUser(other.authUser), peerAddr(other.peerAddr), context(context) { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 39938f4eb9..5d816ba5bb 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -87,7 +87,7 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& vCommands.push_back(make_pair(entry.second.front()->category + entry.first, entry.second.front())); sort(vCommands.begin(), vCommands.end()); - JSONRPCRequest jreq(helpreq); + JSONRPCRequest jreq = helpreq; jreq.mode = JSONRPCRequest::GET_HELP; jreq.params = UniValue(); @@ -137,8 +137,9 @@ static RPCHelpMan help() { {"command", RPCArg::Type::STR, /* default */ "all commands", "The command to get help on"}, }, - RPCResult{ - RPCResult::Type::STR, "", "The help text" + { + RPCResult{RPCResult::Type::STR, "", "The help text"}, + RPCResult{RPCResult::Type::ANY, "", ""}, }, RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue @@ -494,7 +495,7 @@ std::vector<std::string> CRPCTable::listCommands() const UniValue CRPCTable::dumpArgMap(const JSONRPCRequest& args_request) const { - JSONRPCRequest request(args_request); + JSONRPCRequest request = args_request; request.mode = JSONRPCRequest::GET_ARGS; UniValue ret{UniValue::VARR}; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 44e58cb75f..ba3105ca01 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,17 +113,80 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } +namespace { + +/** + * Quote an argument for shell. + * + * @note This is intended for help, not for security-sensitive purposes. + */ +std::string ShellQuote(const std::string& s) +{ + std::string result; + result.reserve(s.size() * 2); + for (const char ch: s) { + if (ch == '\'') { + result += "'\''"; + } else { + result += ch; + } + } + return "'" + result + "'"; +} + +/** + * Shell-quotes the argument if it needs quoting, else returns it literally, to save typing. + * + * @note This is intended for help, not for security-sensitive purposes. + */ +std::string ShellQuoteIfNeeded(const std::string& s) +{ + for (const char ch: s) { + if (ch == ' ' || ch == '\'' || ch == '"') { + return ShellQuote(s); + } + } + + return s; +} + +} + std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; } +std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& args) +{ + std::string result = "> bitcoin-cli -named " + methodname; + for (const auto& argpair: args) { + const auto& value = argpair.second.isStr() + ? argpair.second.get_str() + : argpair.second.write(); + result += " " + argpair.first + "=" + ShellQuoteIfNeeded(value); + } + result += "\n"; + return result; +} + std::string HelpExampleRpc(const std::string& methodname, const std::string& args) { return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", " "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; } +std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& args) +{ + UniValue params(UniValue::VOBJ); + for (const auto& param: args) { + params.pushKV(param.first, param.second); + } + + return "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", " + "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; +} + // Converts a hex string to a public key if possible CPubKey HexToPubKey(const std::string& hex_in) { @@ -442,6 +505,7 @@ std::string RPCResults::ToDescriptionString() const { std::string result; for (const auto& r : m_results) { + if (r.m_type == RPCResult::Type::ANY) continue; // for testing only if (r.m_cond.empty()) { result += "\nResult:\n"; } else { @@ -459,7 +523,7 @@ std::string RPCExamples::ToDescriptionString() const return m_examples.empty() ? m_examples : "\nExamples:\n" + m_examples; } -UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) +UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const { if (request.mode == JSONRPCRequest::GET_ARGS) { return GetArgMap(); @@ -471,7 +535,9 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) if (request.mode == JSONRPCRequest::GET_HELP || !IsValidNumArgs(request.params.size())) { throw std::runtime_error(ToString()); } - return m_fun(*this, request); + const UniValue ret = m_fun(*this, request); + CHECK_NONFATAL(std::any_of(m_results.m_results.begin(), m_results.m_results.end(), [ret](const RPCResult& res) { return res.MatchesType(ret); })); + return ret; } bool RPCHelpMan::IsValidNumArgs(size_t num_args) const @@ -677,6 +743,9 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const sections.PushSection({indent + "..." + maybe_separator, m_description}); return; } + case Type::ANY: { + CHECK_NONFATAL(false); // Only for testing + } case Type::NONE: { sections.PushSection({indent + "null" + maybe_separator, Description("json null")}); return; @@ -742,6 +811,42 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const CHECK_NONFATAL(false); } +bool RPCResult::MatchesType(const UniValue& result) const +{ + switch (m_type) { + case Type::ELISION: { + return false; + } + case Type::ANY: { + return true; + } + case Type::NONE: { + return UniValue::VNULL == result.getType(); + } + case Type::STR: + case Type::STR_HEX: { + return UniValue::VSTR == result.getType(); + } + case Type::NUM: + case Type::STR_AMOUNT: + case Type::NUM_TIME: { + return UniValue::VNUM == result.getType(); + } + case Type::BOOL: { + return UniValue::VBOOL == result.getType(); + } + case Type::ARR_FIXED: + case Type::ARR: { + return UniValue::VARR == result.getType(); + } + case Type::OBJ_DYN: + case Type::OBJ: { + return UniValue::VOBJ == result.getType(); + } + } // no default case, so the compiler can warn about missing cases + CHECK_NONFATAL(false); +} + std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; diff --git a/src/rpc/util.h b/src/rpc/util.h index 94c2d2d626..6e783e19fb 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -78,8 +78,12 @@ extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strNa extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); extern CAmount AmountFromValue(const UniValue& value); + +using RPCArgList = std::vector<std::pair<std::string, UniValue>>; extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); +extern std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); +extern std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& args); CPubKey HexToPubKey(const std::string& hex_in); CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& addr_in); @@ -223,6 +227,7 @@ struct RPCResult { NUM, BOOL, NONE, + ANY, //!< Special type to disable type checks (for testing only) STR_AMOUNT, //!< Special string to represent a floating point amount STR_HEX, //!< Special string with only hex chars OBJ_DYN, //!< Special dictionary with keys that are not literals @@ -295,6 +300,8 @@ struct RPCResult { std::string ToStringObj() const; /** Return the description string, including the result type. */ std::string ToDescriptionString() const; + /** Check whether the result JSON type matches. */ + bool MatchesType(const UniValue& result) const; }; struct RPCResults { @@ -333,7 +340,7 @@ public: using RPCMethodImpl = std::function<UniValue(const RPCHelpMan&, const JSONRPCRequest&)>; RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); - UniValue HandleRequest(const JSONRPCRequest& request); + UniValue HandleRequest(const JSONRPCRequest& request) const; std::string ToString() const; /** Return the named args that need to be converted from string to another JSON type */ UniValue GetArgMap() const; diff --git a/src/sync.h b/src/sync.h index 53213c2089..146c228592 100644 --- a/src/sync.h +++ b/src/sync.h @@ -56,7 +56,7 @@ std::string LocksHeld(); template <typename MutexType> void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs); template <typename MutexType> -void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs); +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) LOCKS_EXCLUDED(cs); void DeleteLock(void* cs); bool LockStackEmpty(); @@ -74,7 +74,7 @@ inline void CheckLastCritical(void* cs, std::string& lockname, const char* guard template <typename MutexType> inline void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs) {} template <typename MutexType> -void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) {} +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) LOCKS_EXCLUDED(cs) {} inline void DeleteLock(void* cs) {} inline bool LockStackEmpty() { return true; } #endif diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 04da10715f..9903ba75cb 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -62,7 +62,7 @@ CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(::ChainstateActive(), *m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; block.hashPrevBlock = prev->GetBlockHash(); block.nTime = prev->nTime + 1; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 7557d4618a..e7cb12dbba 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -22,6 +22,7 @@ #include <test/util/setup_common.h> +#include <array> #include <stdint.h> #include <boost/test/unit_test.hpp> @@ -209,46 +210,94 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) { const CChainParams& chainparams = Params(); auto banman = std::make_unique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<CConnmanTest>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool, false); + CNetAddr tor_netaddr; + BOOST_REQUIRE( + tor_netaddr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion")); + const CService tor_service{tor_netaddr, Params().GetDefaultPort()}; + + const std::array<CAddress, 3> addr{CAddress{ip(0xa0b0c001), NODE_NONE}, + CAddress{ip(0xa0b0c002), NODE_NONE}, + CAddress{tor_service, NODE_NONE}}; + + const CNetAddr other_addr{ip(0xa0b0ff01)}; // Not any of addr[]. + + std::array<CNode*, 3> nodes; + banman->ClearBanned(); - CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, INVALID_SOCKET, addr1, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); - dummyNode1.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode1); - dummyNode1.fSuccessfullyConnected = true; - peerLogic->Misbehaving(dummyNode1.GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); // Should be discouraged + nodes[0] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[0], /* nKeyedNetGroupIn */ 0, + /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", + ConnectionType::INBOUND, /* inbound_onion */ false}; + nodes[0]->SetCommonVersion(PROTOCOL_VERSION); + peerLogic->InitializeNode(nodes[0]); + nodes[0]->fSuccessfullyConnected = true; + connman->AddNode(*nodes[0]); + peerLogic->Misbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); // Should be discouraged { - LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); + LOCK(nodes[0]->cs_sendProcessing); + BOOST_CHECK(peerLogic->SendMessages(nodes[0])); } - BOOST_CHECK(banman->IsDiscouraged(addr1)); - BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged - - CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, INVALID_SOCKET, addr2, /* nKeyedNetGroupIn */ 1, /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); - dummyNode2.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode2); - dummyNode2.fSuccessfullyConnected = true; - peerLogic->Misbehaving(dummyNode2.GetId(), DISCOURAGEMENT_THRESHOLD - 1, /* message */ ""); + BOOST_CHECK(banman->IsDiscouraged(addr[0])); + BOOST_CHECK(nodes[0]->fDisconnect); + BOOST_CHECK(!banman->IsDiscouraged(other_addr)); // Different address, not discouraged + + nodes[1] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[1], /* nKeyedNetGroupIn */ 1, + /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", + ConnectionType::INBOUND, /* inbound_onion */ false}; + nodes[1]->SetCommonVersion(PROTOCOL_VERSION); + peerLogic->InitializeNode(nodes[1]); + nodes[1]->fSuccessfullyConnected = true; + connman->AddNode(*nodes[1]); + peerLogic->Misbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1, /* message */ ""); { - LOCK(dummyNode2.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); + LOCK(nodes[1]->cs_sendProcessing); + BOOST_CHECK(peerLogic->SendMessages(nodes[1])); } - BOOST_CHECK(!banman->IsDiscouraged(addr2)); // 2 not discouraged yet... - BOOST_CHECK(banman->IsDiscouraged(addr1)); // ... but 1 still should be - peerLogic->Misbehaving(dummyNode2.GetId(), 1, /* message */ ""); // 2 reaches discouragement threshold + // [0] is still discouraged/disconnected. + BOOST_CHECK(banman->IsDiscouraged(addr[0])); + BOOST_CHECK(nodes[0]->fDisconnect); + // [1] is not discouraged/disconnected yet. + BOOST_CHECK(!banman->IsDiscouraged(addr[1])); + BOOST_CHECK(!nodes[1]->fDisconnect); + peerLogic->Misbehaving(nodes[1]->GetId(), 1, /* message */ ""); // [1] reaches discouragement threshold { - LOCK(dummyNode2.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); + LOCK(nodes[1]->cs_sendProcessing); + BOOST_CHECK(peerLogic->SendMessages(nodes[1])); } - BOOST_CHECK(banman->IsDiscouraged(addr1)); // Expect both 1 and 2 - BOOST_CHECK(banman->IsDiscouraged(addr2)); // to be discouraged now - - peerLogic->FinalizeNode(dummyNode1); - peerLogic->FinalizeNode(dummyNode2); + // Expect both [0] and [1] to be discouraged/disconnected now. + BOOST_CHECK(banman->IsDiscouraged(addr[0])); + BOOST_CHECK(nodes[0]->fDisconnect); + BOOST_CHECK(banman->IsDiscouraged(addr[1])); + BOOST_CHECK(nodes[1]->fDisconnect); + + // Make sure non-IP peers are discouraged and disconnected properly. + + nodes[2] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[2], /* nKeyedNetGroupIn */ 1, + /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", + ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false}; + nodes[2]->SetCommonVersion(PROTOCOL_VERSION); + peerLogic->InitializeNode(nodes[2]); + nodes[2]->fSuccessfullyConnected = true; + connman->AddNode(*nodes[2]); + peerLogic->Misbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); + { + LOCK(nodes[2]->cs_sendProcessing); + BOOST_CHECK(peerLogic->SendMessages(nodes[2])); + } + BOOST_CHECK(banman->IsDiscouraged(addr[0])); + BOOST_CHECK(banman->IsDiscouraged(addr[1])); + BOOST_CHECK(banman->IsDiscouraged(addr[2])); + BOOST_CHECK(nodes[0]->fDisconnect); + BOOST_CHECK(nodes[1]->fDisconnect); + BOOST_CHECK(nodes[2]->fDisconnect); + + for (CNode* node : nodes) { + peerLogic->FinalizeNode(*node); + } + connman->ClearNodes(); } BOOST_AUTO_TEST_CASE(DoS_bantime) diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index d951bda20f..b21d2eae79 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -230,11 +230,8 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) // consensus/tx_verify.cpp:171: bool Consensus::CheckTxInputs(const CTransaction &, TxValidationState &, const CCoinsViewCache &, int, CAmount &): Assertion `!coin.IsSpent()' failed. return; } - try { - (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); - assert(MoneyRange(tx_fee_out)); - } catch (const std::runtime_error&) { - } + (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); + assert(MoneyRange(tx_fee_out)); }, [&] { const CTransaction transaction{random_mutable_transaction}; diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 561230707c..387f9c069c 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -5,13 +5,11 @@ #include <pubkey.h> #include <script/interpreter.h> #include <streams.h> +#include <test/util/script.h> #include <version.h> #include <test/fuzz/fuzz.h> -/** Flags that are not forbidden by an assert */ -static bool IsValidFlagCombination(unsigned flags); - void initialize_script_flags() { static const ECCVerifyHandle verify_handle; @@ -74,10 +72,3 @@ FUZZ_TARGET_INIT(script_flags, initialize_script_flags) return; } } - -static bool IsValidFlagCombination(unsigned flags) -{ - if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) return false; - if (flags & SCRIPT_VERIFY_WITNESS && ~flags & SCRIPT_VERIFY_P2SH) return false; - return true; -} diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 7b57c5dfd8..6b86c8889d 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -6,6 +6,8 @@ #include <script/interpreter.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/script.h> #include <cstdint> #include <limits> @@ -56,17 +58,12 @@ FUZZ_TARGET_INIT(signature_checker, initialize_signature_checker) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const SigVersion sig_version = fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}); - const std::string script_string_1 = fuzzed_data_provider.ConsumeRandomLengthString(65536); - const std::vector<uint8_t> script_bytes_1{script_string_1.begin(), script_string_1.end()}; - const std::string script_string_2 = fuzzed_data_provider.ConsumeRandomLengthString(65536); - const std::vector<uint8_t> script_bytes_2{script_string_2.begin(), script_string_2.end()}; + const auto script_1 = ConsumeScript(fuzzed_data_provider, 65536); + const auto script_2 = ConsumeScript(fuzzed_data_provider, 65536); std::vector<std::vector<unsigned char>> stack; - (void)EvalScript(stack, {script_bytes_1.begin(), script_bytes_1.end()}, flags, FuzzedSignatureChecker(fuzzed_data_provider), sig_version, nullptr); - if ((flags & SCRIPT_VERIFY_CLEANSTACK) != 0 && ((flags & SCRIPT_VERIFY_P2SH) == 0 || (flags & SCRIPT_VERIFY_WITNESS) == 0)) { + (void)EvalScript(stack, script_1, flags, FuzzedSignatureChecker(fuzzed_data_provider), sig_version, nullptr); + if (!IsValidFlagCombination(flags)) { return; } - if ((flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { - return; - } - (void)VerifyScript({script_bytes_1.begin(), script_bytes_1.end()}, {script_bytes_2.begin(), script_bytes_2.end()}, nullptr, flags, FuzzedSignatureChecker(fuzzed_data_provider), nullptr); + (void)VerifyScript(script_1, script_2, nullptr, flags, FuzzedSignatureChecker(fuzzed_data_provider), nullptr); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 93418ab1ff..d786ac1db1 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -80,7 +80,7 @@ CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, co CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length, const bool maybe_p2wsh) noexcept { - const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CScript r_script{b.begin(), b.end()}; if (maybe_p2wsh && fuzzed_data_provider.ConsumeBool()) { uint256 script_hash; diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9acd17c463..9ba004cc38 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -44,7 +44,7 @@ BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params) options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; options.blockMinFeeRate = blockMinFeeRate; - return BlockAssembler(*m_node.mempool, params, options); + return BlockAssembler(::ChainstateActive(), *m_node.mempool, params, options); } constexpr static struct { @@ -122,7 +122,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co uint256 hashHighFeeTx = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4U); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); @@ -143,7 +143,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; uint256 hashLowFeeTx = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(feeToUse).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx); @@ -157,7 +157,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -179,7 +179,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; uint256 hashLowFeeTx2 = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that this tx isn't selected. for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { @@ -192,7 +192,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vin[0].prevout.n = 1; tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee m_node.mempool->addUnchecked(entry.Fee(10000).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey); + pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 9U); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) fCheckpointsEnabled = false; // Simple block creation, nothing special yet: - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // We can't make transactions until we have inputs // Therefore, load 110 blocks :) @@ -252,7 +252,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) LOCK(m_node.mempool->cs); // Just to make sure we can still make simple blocks - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); const CAmount BLOCKSUBSIDY = 50*COIN; const CAmount LOWFEE = CENT; @@ -277,7 +277,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); m_node.mempool->clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -291,7 +291,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); m_node.mempool->clear(); // block size > limit @@ -311,13 +311,13 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); m_node.mempool->clear(); // orphan in *m_node.mempool, template creation fails hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); m_node.mempool->clear(); // child with higher feerate than parent @@ -334,7 +334,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].nValue = tx.vout[0].nValue+BLOCKSUBSIDY-HIGHERFEE; //First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); m_node.mempool->clear(); // coinbase in *m_node.mempool, template creation fails @@ -346,7 +346,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // give it a fee so it'll get mined m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); // Should throw bad-cb-multiple - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); m_node.mempool->clear(); // double spend txn pair in *m_node.mempool, template creation fails @@ -359,7 +359,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); m_node.mempool->clear(); // subsidy changing @@ -375,7 +375,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); ::ChainActive().SetTip(next); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // Extend to a 210000-long block chain. while (::ChainActive().Tip()->nHeight < 210000) { CBlockIndex* prev = ::ChainActive().Tip(); @@ -387,7 +387,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) next->BuildSkip(); ::ChainActive().SetTip(next); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // invalid p2sh txn in *m_node.mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -404,7 +404,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); // Should throw block-validation-failed - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); m_node.mempool->clear(); // Delete the dummy blocks again. @@ -492,7 +492,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // None of the of the absolute height/time locked tx should have made // it into the template because we still check IsFinalTx in CreateNewBlock, @@ -505,7 +505,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) ::ChainActive().Tip()->nHeight++; SetMockTime(::ChainActive().Tip()->GetMedianTimePast() + 1); - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5U); ::ChainActive().Tip()->nHeight--; diff --git a/src/test/ref_tests.cpp b/src/test/ref_tests.cpp deleted file mode 100644 index 0ec0799fbc..0000000000 --- a/src/test/ref_tests.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <util/ref.h> - -#include <boost/test/unit_test.hpp> - -BOOST_AUTO_TEST_SUITE(ref_tests) - -BOOST_AUTO_TEST_CASE(ref_test) -{ - util::Ref ref; - BOOST_CHECK(!ref.Has<int>()); - BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); - int value = 5; - ref.Set(value); - BOOST_CHECK(ref.Has<int>()); - BOOST_CHECK_EQUAL(ref.Get<int>(), 5); - ++ref.Get<int>(); - BOOST_CHECK_EQUAL(ref.Get<int>(), 6); - BOOST_CHECK_EQUAL(value, 6); - ++value; - BOOST_CHECK_EQUAL(value, 7); - BOOST_CHECK_EQUAL(ref.Get<int>(), 7); - BOOST_CHECK(!ref.Has<bool>()); - BOOST_CHECK_THROW(ref.Get<bool>(), NonFatalCheckError); - ref.Clear(); - BOOST_CHECK(!ref.Has<int>()); - BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 2e0fc7e48f..7c50c72041 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -10,9 +10,10 @@ #include <interfaces/chain.h> #include <node/context.h> #include <test/util/setup_common.h> -#include <util/ref.h> #include <util/time.h> +#include <any> + #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> @@ -32,7 +33,7 @@ UniValue RPCTestingSetup::CallRPC(std::string args) boost::split(vArgs, args, boost::is_any_of(" \t")); std::string strMethod = vArgs[0]; vArgs.erase(vArgs.begin()); - util::Ref context{m_node}; + std::any context{&m_node}; JSONRPCRequest request(context); request.strMethod = strMethod; request.params = RPCConvertValues(strMethod, vArgs); @@ -430,4 +431,39 @@ BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight) } } +BOOST_AUTO_TEST_CASE(help_example) +{ + // test different argument types + const RPCArgList& args = {{"foo", "bar"}, {"b", true}, {"n", 1}}; + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", args), "> bitcoin-cli -named test foo=bar b=true n=1\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + + // test shell escape + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b'ar"}}), "> bitcoin-cli -named test foo='b'''ar'\n"); + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b\"ar"}}), "> bitcoin-cli -named test foo='b\"ar'\n"); + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b ar"}}), "> bitcoin-cli -named test foo='b ar'\n"); + + // test object params + UniValue obj_value(UniValue::VOBJ); + obj_value.pushKV("foo", "bar"); + obj_value.pushKV("b", false); + obj_value.pushKV("n", 1); + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", obj_value}}), "> bitcoin-cli -named test name='{\"foo\":\"bar\",\"b\":false,\"n\":1}'\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + + // test array params + UniValue arr_value(UniValue::VARR); + arr_value.push_back("bar"); + arr_value.push_back(false); + arr_value.push_back(1); + BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", arr_value}}), "> bitcoin-cli -named test name='[\"bar\",false,1]'\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"1.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"); + + // test types don't matter for shell + BOOST_CHECK_EQUAL(HelpExampleCliNamed("foo", {{"arg", true}}), HelpExampleCliNamed("foo", {{"arg", "true"}})); + + // test types matter for Rpc + BOOST_CHECK_NE(HelpExampleRpcNamed("foo", {{"arg", true}}), HelpExampleRpcNamed("foo", {{"arg", "true"}})); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index ba1edba0ae..3fc3329da2 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -41,8 +41,8 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { auto block = std::make_shared<CBlock>( - BlockAssembler{*Assert(node.mempool), Params()} - .CreateNewBlock(::ChainstateActive(), coinbase_scriptPubKey) + BlockAssembler{::ChainstateActive(), *Assert(node.mempool), Params()} + .CreateNewBlock(coinbase_scriptPubKey) ->block); LOCK(cs_main); diff --git a/src/test/util/script.cpp b/src/test/util/script.cpp new file mode 100644 index 0000000000..a5852daa60 --- /dev/null +++ b/src/test/util/script.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/interpreter.h> +#include <test/util/script.h> + +bool IsValidFlagCombination(unsigned flags) +{ + if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) return false; + if (flags & SCRIPT_VERIFY_WITNESS && ~flags & SCRIPT_VERIFY_P2SH) return false; + return true; +} diff --git a/src/test/util/script.h b/src/test/util/script.h index abd14c2067..428b3e10b3 100644 --- a/src/test/util/script.h +++ b/src/test/util/script.h @@ -18,4 +18,7 @@ static const CScript P2WSH_OP_TRUE{ return hash; }())}; +/** Flags that are not forbidden by an assert in script validation */ +bool IsValidFlagCombination(unsigned flags); + #endif // BITCOIN_TEST_UTIL_SCRIPT_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index bfb3466dcf..f7800aefca 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -247,7 +247,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa { const CChainParams& chainparams = Params(); CTxMemPool empty_pool; - CBlock block = BlockAssembler(empty_pool, chainparams).CreateNewBlock(::ChainstateActive(), scriptPubKey)->block; + CBlock block = BlockAssembler(::ChainstateActive(), empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; Assert(block.vtx.size() == 1); for (const CMutableTransaction& tx : txns) { diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index f3fc83078f..552be0a2da 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -63,7 +63,7 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) static int i = 0; static uint64_t time = Params().GenesisBlock().nTime; - auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(::ChainstateActive(), CScript{} << i++ << OP_TRUE); + auto ptemplate = BlockAssembler(m_node.chainman->ActiveChainstate(), *m_node.mempool, Params()).CreateNewBlock(CScript{} << i++ << OP_TRUE); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; @@ -325,7 +325,7 @@ BOOST_AUTO_TEST_CASE(witness_commitment_index) { CScript pubKey; pubKey << 1 << OP_TRUE; - auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(::ChainstateActive(), pubKey); + auto ptemplate = BlockAssembler(m_node.chainman->ActiveChainstate(), *m_node.mempool, Params()).CreateNewBlock(pubKey); CBlock pblock = ptemplate->block; CTxOut witness; diff --git a/src/txorphanage.h b/src/txorphanage.h index df55cdb3be..dc9fa87dbb 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -24,7 +24,7 @@ public: bool AddTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); /** Check if we already have an orphan transaction (by txid or wtxid) */ - bool HaveTx(const GenTxid& gtxid) const EXCLUSIVE_LOCKS_REQUIRED(!g_cs_orphans); + bool HaveTx(const GenTxid& gtxid) const LOCKS_EXCLUDED(::g_cs_orphans); /** Get an orphan transaction and its orginating peer * (Transaction ref will be nullptr if not found) @@ -38,7 +38,7 @@ public: void EraseForPeer(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); /** Erase all orphans included in or invalidated by a new block */ - void EraseForBlock(const CBlock& block) EXCLUSIVE_LOCKS_REQUIRED(!g_cs_orphans); + void EraseForBlock(const CBlock& block) LOCKS_EXCLUDED(::g_cs_orphans); /** Limit the orphanage to the given maximum */ unsigned int LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); diff --git a/src/util/ref.h b/src/util/ref.h deleted file mode 100644 index 9685ea9fec..0000000000 --- a/src/util/ref.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_REF_H -#define BITCOIN_UTIL_REF_H - -#include <util/check.h> - -#include <typeindex> - -namespace util { - -/** - * Type-safe dynamic reference. - * - * This implements a small subset of the functionality in C++17's std::any - * class, and can be dropped when the project updates to C++17 - * (https://github.com/bitcoin/bitcoin/issues/16684) - */ -class Ref -{ -public: - Ref() = default; - template<typename T> Ref(T& value) { Set(value); } - template<typename T> T& Get() const { CHECK_NONFATAL(Has<T>()); return *static_cast<T*>(m_value); } - template<typename T> void Set(T& value) { m_value = &value; m_type = std::type_index(typeid(T)); } - template<typename T> bool Has() const { return m_value && m_type == std::type_index(typeid(T)); } - void Clear() { m_value = nullptr; m_type = std::type_index(typeid(void)); } - -private: - void* m_value = nullptr; - std::type_index m_type = std::type_index(typeid(void)); -}; - -} // namespace util - -#endif // BITCOIN_UTIL_REF_H diff --git a/src/util/system.h b/src/util/system.h index 291f3f5541..29657e56e2 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -25,6 +25,7 @@ #include <util/threadnames.h> #include <util/time.h> +#include <any> #include <exception> #include <map> #include <optional> @@ -500,6 +501,18 @@ inline void insert(std::set<TsetT>& dst, const Tsrc& src) { dst.insert(src.begin(), src.end()); } +/** + * Helper function to access the contained object of a std::any instance. + * Returns a pointer to the object if passed instance has a value and the type + * matches, nullptr otherwise. + */ +template<typename T> +T* AnyPtr(const std::any& any) noexcept +{ + T* const* ptr = std::any_cast<T*>(&any); + return ptr ? *ptr : nullptr; +} + #ifdef WIN32 class WinCmdLineArgs { diff --git a/src/validation.cpp b/src/validation.cpp index d1b9efe7ba..19363c0efb 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -169,7 +169,7 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const { AssertLockHeld(cs_main); assert(std::addressof(g_chainman.BlockIndex()) == std::addressof(m_block_index)); @@ -690,7 +690,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } - // Bring the best block into scope + // This is const, but calls into the back end CoinsViews. The CCoinsViewDB at the bottom of the + // hierarchy brings the best block into scope. See CCoinsViewDB::GetBestBlock(). m_view.GetBestBlock(); // we have all inputs cached now, so switch back to dummy (to protect @@ -1099,9 +1100,9 @@ static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainp assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); const MempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptSingleTransaction(tx, args); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { - // Remove coins that were not present in the coins cache before calling ATMPW; - // this is to prevent memory DoS in case we receive a large number of - // invalid transactions that attempt to overrun the in-memory coins cache + // Remove coins that were not present in the coins cache before calling + // AcceptSingleTransaction(); this is to prevent memory DoS in case we receive a large + // number of invalid transactions that attempt to overrun the in-memory coins cache // (`CCoinsViewCache::cacheCoins`). for (const COutPoint& hashTx : coins_to_uncache) diff --git a/src/validation.h b/src/validation.h index 2ff5f4ac87..21e63947fa 100644 --- a/src/validation.h +++ b/src/validation.h @@ -457,7 +457,7 @@ public: const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Find the last common block between the parameter chain and a locator. */ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index ada586119a..da5b84ce83 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -15,7 +15,6 @@ #include <sync.h> #include <uint256.h> #include <util/check.h> -#include <util/ref.h> #include <util/system.h> #include <util/ui_change_type.h> #include <wallet/context.h> @@ -515,7 +514,7 @@ public: { for (const CRPCCommand& command : GetWalletRPCCommands()) { m_rpc_commands.emplace_back(command.category, command.name, [this, &command](const JSONRPCRequest& request, UniValue& result, bool last_handler) { - return command.actor({request, m_context}, result, last_handler); + return command.actor({request, &m_context}, result, last_handler); }, command.argNames, command.unique_id); m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 0737e68cf3..769987ccf2 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1737,22 +1737,23 @@ RPCHelpMan listdescriptors() "listdescriptors", "\nList descriptors imported into a descriptor-enabled wallet.", {}, - RPCResult{ - RPCResult::Type::ARR, "", "Response is an array of descriptor objects", + RPCResult{RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "wallet_name", "Name of wallet this operation was performed on"}, + {RPCResult::Type::ARR, "descriptors", "Array of descriptor objects", { {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "desc", "Descriptor string representation"}, {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, {RPCResult::Type::BOOL, "active", "Activeness flag"}, - {RPCResult::Type::BOOL, "internal", true, "Whether this is internal or external descriptor; defined only for active descriptors"}, + {RPCResult::Type::BOOL, "internal", true, "Whether this is an internal or external descriptor; defined only for active descriptors"}, {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { {RPCResult::Type::NUM, "", "Range start inclusive"}, {RPCResult::Type::NUM, "", "Range end inclusive"}, }}, {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, }}, - } - }, + }} + }}, RPCExamples{ HelpExampleCli("listdescriptors", "") + HelpExampleRpc("listdescriptors", "") }, @@ -1769,7 +1770,7 @@ RPCHelpMan listdescriptors() LOCK(wallet->cs_wallet); - UniValue response(UniValue::VARR); + UniValue descriptors(UniValue::VARR); const auto active_spk_mans = wallet->GetActiveScriptPubKeyMans(); for (const auto& spk_man : wallet->GetAllScriptPubKeyMans()) { const auto desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); @@ -1798,9 +1799,13 @@ RPCHelpMan listdescriptors() spk.pushKV("range", range); spk.pushKV("next", wallet_descriptor.next_index); } - response.push_back(spk); + descriptors.push_back(spk); } + UniValue response(UniValue::VOBJ); + response.pushKV("wallet_name", wallet->GetName()); + response.pushKV("descriptors", descriptors); + return response; }, }; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e70bbafde0..45f292055e 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -22,7 +22,6 @@ #include <util/fees.h> #include <util/message.h> // For MessageSign() #include <util/moneystr.h> -#include <util/ref.h> #include <util/string.h> #include <util/system.h> #include <util/translation.h> @@ -124,12 +123,13 @@ void EnsureWalletIsUnlocked(const CWallet& wallet) } } -WalletContext& EnsureWalletContext(const util::Ref& context) +WalletContext& EnsureWalletContext(const std::any& context) { - if (!context.Has<WalletContext>()) { + auto wallet_context = util::AnyPtr<WalletContext>(context); + if (!wallet_context) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Wallet context not found"); } - return context.Get<WalletContext>(); + return *wallet_context; } // also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank @@ -2711,6 +2711,8 @@ static RPCHelpMan createwallet() RPCExamples{ HelpExampleCli("createwallet", "\"testwallet\"") + HelpExampleRpc("createwallet", "\"testwallet\"") + + HelpExampleCliNamed("createwallet", {{"wallet_name", "descriptors"}, {"avoid_reuse", true}, {"descriptors", true}, {"load_on_startup", true}}) + + HelpExampleRpcNamed("createwallet", {{"wallet_name", "descriptors"}, {"avoid_reuse", true}, {"descriptors", true}, {"load_on_startup", true}}) }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -3379,7 +3381,7 @@ RPCHelpMan signrawtransactionwithwallet() static RPCHelpMan bumpfee_helper(std::string method_name) { - bool want_psbt = method_name == "psbtbumpfee"; + const bool want_psbt = method_name == "psbtbumpfee"; const std::string incremental_fee{CFeeRate(DEFAULT_INCREMENTAL_RELAY_FEE).ToString(FeeEstimateMode::SAT_VB)}; return RPCHelpMan{method_name, @@ -3413,14 +3415,14 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "still be replaceable in practice, for example if it has unconfirmed ancestors which\n" "are replaceable).\n"}, {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, + "\"" + FeeModes("\"\n\"") + "\""}, }, "options"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { - {RPCResult::Type::STR, "psbt", "The base64-encoded unsigned PSBT of the new transaction." + std::string(want_psbt ? "" : " Only returned when wallet private keys are disabled. (DEPRECATED)")}, + {RPCResult::Type::STR, "psbt", "The base64-encoded unsigned PSBT of the new transaction."}, }, want_psbt ? std::vector<RPCResult>{} : std::vector<RPCResult>{{RPCResult::Type::STR_HEX, "txid", "The id of the new transaction. Only returned when wallet private keys are enabled."}} ), @@ -3437,7 +3439,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "\nBump the fee, get the new transaction\'s" + std::string(want_psbt ? "psbt" : "txid") + "\n" + HelpExampleCli(method_name, "<txid>") }, - [want_psbt](const RPCHelpMan& self, const JSONRPCRequest& request) mutable -> UniValue + [want_psbt](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index b82fe1ec76..8b88ffe8ed 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -7,6 +7,7 @@ #include <span.h> +#include <any> #include <memory> #include <string> #include <vector> @@ -31,7 +32,7 @@ Span<const CRPCCommand> GetWalletRPCCommands(); std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request); void EnsureWalletIsUnlocked(const CWallet&); -WalletContext& EnsureWalletContext(const util::Ref& context); +WalletContext& EnsureWalletContext(const std::any& context); LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); RPCHelpMan getaddressinfo(); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 30cc452065..ba2e17d62a 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -4,6 +4,7 @@ #include <wallet/wallet.h> +#include <any> #include <future> #include <memory> #include <stdint.h> @@ -15,7 +16,6 @@ #include <rpc/server.h> #include <test/util/logging.h> #include <test/util/setup_common.h> -#include <util/ref.h> #include <util/translation.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -213,7 +213,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); - util::Ref context; + std::any context; JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(keys); @@ -265,7 +265,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) AddWallet(wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } - util::Ref context; + std::any context; JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); @@ -281,7 +281,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); - util::Ref context; + std::any context; JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6031d36b97..192fd60dc2 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1181,7 +1181,7 @@ public: * Obviously holding cs_main/cs_wallet when going into this call may cause * deadlock */ - void BlockUntilSyncedToCurrentChain() const EXCLUSIVE_LOCKS_REQUIRED(!::cs_main, !cs_wallet); + void BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(::cs_main) EXCLUSIVE_LOCKS_REQUIRED(!cs_wallet); /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index bdbfa5aed1..c7981d31dc 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -6,11 +6,11 @@ Connect to a single node. Generate 2 blocks (save the coinbases for later). -Generate 427 more blocks. -[Policy/Consensus] Check that NULLDUMMY compliant transactions are accepted in the 430th block. +Generate COINBASE_MATURITY (CB) more blocks to ensure the coinbases are mature. +[Policy/Consensus] Check that NULLDUMMY compliant transactions are accepted in block CB + 3. [Policy] Check that non-NULLDUMMY transactions are rejected before activation. -[Consensus] Check that the new NULLDUMMY rules are not enforced on the 431st block. -[Policy/Consensus] Check that the new NULLDUMMY rules are enforced on the 432nd block. +[Consensus] Check that the new NULLDUMMY rules are not enforced on block CB + 4. +[Policy/Consensus] Check that the new NULLDUMMY rules are enforced on block CB + 5. """ import time @@ -20,13 +20,14 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +COINBASE_MATURITY = 100 NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" def trueDummy(tx): scriptSig = CScript(tx.vin[0].scriptSig) newscript = [] for i in scriptSig: - if (len(newscript) == 0): + if len(newscript) == 0: assert len(i) == 0 newscript.append(b'\x51') else: @@ -37,13 +38,13 @@ def trueDummy(tx): class NULLDUMMYTest(BitcoinTestFramework): def set_test_params(self): - # Need two nodes only so GBT doesn't complain that it's not connected + # Need two nodes so GBT (getblocktemplate) doesn't complain that it's not connected. self.num_nodes = 2 self.setup_clean_chain = True # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through # normal segwit activation here (and don't use the default always-on behaviour). self.extra_args = [[ - '-segwitheight=432', + f'-segwitheight={COINBASE_MATURITY + 5}', '-addresstype=legacy', ]] * 2 @@ -64,16 +65,16 @@ class NULLDUMMYTest(BitcoinTestFramework): wmulti.importaddress(self.ms_address) wmulti.importaddress(self.wit_ms_address) - self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 + self.coinbase_blocks = self.nodes[0].generate(2) # block height = 2 coinbase_txid = [] for i in self.coinbase_blocks: coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0]) - self.nodes[0].generate(427) # Block 429 + self.nodes[0].generate(COINBASE_MATURITY) # block height = COINBASE_MATURITY + 2 self.lastblockhash = self.nodes[0].getbestblockhash() - self.lastblockheight = 429 - self.lastblocktime = int(time.time()) + 429 + self.lastblockheight = COINBASE_MATURITY + 2 + self.lastblocktime = int(time.time()) + self.lastblockheight - self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") + self.log.info(f"Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [{COINBASE_MATURITY + 3}]") test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) @@ -87,7 +88,7 @@ class NULLDUMMYTest(BitcoinTestFramework): trueDummy(test2tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) - self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") + self.log.info(f"Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [{COINBASE_MATURITY + 4}]") self.block_submit(self.nodes[0], [test2tx], False, True) self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") @@ -104,7 +105,7 @@ class NULLDUMMYTest(BitcoinTestFramework): assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test5tx], True) - self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") + self.log.info(f"Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [{COINBASE_MATURITY + 5}]") for i in test6txs: self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test6txs, True, True) @@ -130,5 +131,6 @@ class NULLDUMMYTest(BitcoinTestFramework): else: assert_equal(node.getbestblockhash(), self.lastblockhash) + if __name__ == '__main__': NULLDUMMYTest().main() diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index 8d02949ff4..c1444164ce 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -36,15 +36,16 @@ class ListDescriptorsTest(BitcoinTestFramework): self.log.info('Test the command for empty descriptors wallet.') node.createwallet(wallet_name='w2', blank=True, descriptors=True) - assert_equal(0, len(node.get_wallet_rpc('w2').listdescriptors())) + assert_equal(0, len(node.get_wallet_rpc('w2').listdescriptors()['descriptors'])) self.log.info('Test the command for a default descriptors wallet.') node.createwallet(wallet_name='w3', descriptors=True) result = node.get_wallet_rpc('w3').listdescriptors() - assert_equal(6, len(result)) - assert_equal(6, len([d for d in result if d['active']])) - assert_equal(3, len([d for d in result if d['internal']])) - for item in result: + assert_equal("w3", result['wallet_name']) + assert_equal(6, len(result['descriptors'])) + assert_equal(6, len([d for d in result['descriptors'] if d['active']])) + assert_equal(3, len([d for d in result['descriptors'] if d['internal']])) + for item in result['descriptors']: assert item['desc'] != '' assert item['next'] == 0 assert item['range'] == [0, 0] @@ -59,12 +60,17 @@ class ListDescriptorsTest(BitcoinTestFramework): 'desc': descsum_create('wpkh(' + xprv + hardened_path + '/0/*)'), 'timestamp': 1296688602, }]) - expected = {'desc': descsum_create('wpkh([80002067' + hardened_path + ']' + xpub_acc + '/0/*)'), - 'timestamp': 1296688602, - 'active': False, - 'range': [0, 0], - 'next': 0} - assert_equal([expected], wallet.listdescriptors()) + expected = { + 'wallet_name': 'w2', + 'descriptors': [ + {'desc': descsum_create('wpkh([80002067' + hardened_path + ']' + xpub_acc + '/0/*)'), + 'timestamp': 1296688602, + 'active': False, + 'range': [0, 0], + 'next': 0}, + ], + } + assert_equal(expected, wallet.listdescriptors()) self.log.info('Test non-active non-range combo descriptor') node.createwallet(wallet_name='w4', blank=True, descriptors=True) @@ -73,9 +79,14 @@ class ListDescriptorsTest(BitcoinTestFramework): 'desc': descsum_create('combo(' + node.get_deterministic_priv_key().key + ')'), 'timestamp': 1296688602, }]) - expected = [{'active': False, - 'desc': 'combo(0227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3f)#np574htj', - 'timestamp': 1296688602}] + expected = { + 'wallet_name': 'w4', + 'descriptors': [ + {'active': False, + 'desc': 'combo(0227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3f)#np574htj', + 'timestamp': 1296688602}, + ] + } assert_equal(expected, wallet.listdescriptors()) diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh index b2ed811cda..0ab6aad50c 100755 --- a/test/lint/extended-lint-cppcheck.sh +++ b/test/lint/extended-lint-cppcheck.sh @@ -57,7 +57,6 @@ IGNORED_WARNINGS=( "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." - "src/util/ref.h:.* Class 'Ref' has a constructor with 1 argument that is not explicit." "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." ) diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 3fc9fac25c..5b832b5763 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -28,7 +28,6 @@ race:BerkeleyBatch race:BerkeleyDatabase race:DatabaseBatch race:leveldb::DBImpl::DeleteObsoleteFiles -race:validation_chainstatemanager_tests race:zmq::* race:bitcoin-qt @@ -42,6 +41,10 @@ deadlock:TestPotentialDeadLockDetected race:src/qt/test/* deadlock:src/qt/test/* +# Race in src/test/main.cpp +# Can be removed once upgraded to boost test 1.74 in depends +race:validation_chainstatemanager_tests + # External libraries deadlock:libdb race:libzmq diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 97f0f45e7f..27885b094e 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,6 +1,10 @@ # -fsanitize=undefined suppressions # ================================= -# No suppressions at the moment. Hooray! +# This would be `signed-integer-overflow:CTxMemPool::PrioritiseTransaction`, +# however due to a bug in clang the symbolizer is disabled and thus no symbol +# names can be used. +# See https://github.com/google/sanitizers/issues/1364 +signed-integer-overflow:txmempool.cpp # -fsanitize=integer suppressions # =============================== |