diff options
57 files changed, 1461 insertions, 413 deletions
diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 5f0640ac27..d63ff2b9d5 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -18,7 +18,7 @@ #define CLIENT_VERSION_MAJOR 0 /* Minor version */ -#define CLIENT_VERSION_MINOR 19 +#define CLIENT_VERSION_MINOR 20 /* Build revision */ #define CLIENT_VERSION_REVISION 99 diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index adf4fa0354..6ec40461c2 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -8,6 +8,7 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\util\url.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/configure.ac b/configure.ac index a2d33ff5b6..4c9902efc6 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.69]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 19) +define(_CLIENT_VERSION_MINOR, 20) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -1265,7 +1265,7 @@ if test x$use_pkgconfig = xyes; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) + PKG_CHECK_MODULES([EVENT], [libevent], [use_libevent=yes], [AC_MSG_ERROR(libevent not found.)]) if test x$TARGET_OS != xwindows; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) fi @@ -1285,7 +1285,7 @@ if test x$use_pkgconfig = xyes; then else if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) + AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) if test x$TARGET_OS != xwindows; then AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) @@ -1523,6 +1523,7 @@ AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) +AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index a13a42d391..5d190de54c 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-linux-0.20" +name: "bitcoin-core-linux-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 58531c81b4..37f2a534b8 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-osx-0.20" +name: "bitcoin-core-osx-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index c5eea97c77..0cc1adc557 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-win-0.20" +name: "bitcoin-core-win-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 8500379025..9f99b36f88 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -220,8 +220,6 @@ repository and will likely put one up soon. [guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html [guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html -[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 2daa8aba5e..e20b2a048d 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,33 +13,106 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ -- "$@" } -# Deterministically build Bitcoin Core for HOSTs (overriable by environment) -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + +# Deterministically build Bitcoin Core for HOSTs (overridable by environment) +# shellcheck disable=SC2153 +for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build - trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm' - # shellcheck disable=SC2086 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # shellcheck disable=SC2086 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) done diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ee207a957c..6ef803340b 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash export LC_ALL=C set -e -o pipefail +export TZ=UTC # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." @@ -36,23 +37,41 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Determine output paths to use in CROSS_* environment variables -CROSS_GLIBC="$(store_path glibc-cross-${HOST})" -CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" -CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" -CROSS_GCC="$(store_path gcc-cross-${HOST})" -CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... -CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # Set environment variables to point Guix's cross-toolchain to the right # includes/libs for $HOST -# -# NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because -# the limits.h in it is missing a '#include_next <limits.h>' -# -export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" -export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" -export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" + export CPATH="${NATIVE_GCC}/include" + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because + # the limits.h in it is missing a '#include_next <limits.h>' + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" @@ -74,16 +93,20 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes [ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env # Determine the correct value for -Wl,--dynamic-linker for the current $HOST -glibc_dynamic_linker=$( - case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - *) exit 1 ;; - esac -) +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + *) exit 1 ;; + esac + ) + ;; +esac # Environment variables for determinism export QT_RCC_TEST=1 @@ -136,11 +159,27 @@ DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" # Binary Tarball Building # ########################### -# Similar flags to Gitian -CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" -HOST_CFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_CXXFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +case "$HOST" in + *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; +esac + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" @@ -160,7 +199,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" ${CONFIGFLAGS} \ CFLAGS="${HOST_CFLAGS}" \ CXXFLAGS="${HOST_CXXFLAGS}" \ - LDFLAGS="${HOST_LDFLAGS}" + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -169,9 +208,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} + + case "$HOST" in + *linux*|*mingw*) + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} + ;; + esac + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} + ;; + esac # Setup the directory where our Bitcoin Core build for HOST will be # installed. This directory will also later serve as the input for our @@ -180,9 +231,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + + case "$HOST" in + *mingw*) + cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe + ;; + esac ( cd installed + case "$HOST" in + *mingw*) + mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll + ;; + esac + # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete @@ -196,19 +259,60 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + ;; + esac # Finally, deterministically produce {non-,}debug binary tarballs ready # for release - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + esac ) ) + +case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir unsigned + cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; +esac diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c25ac2977b..86c1a8d27f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -10,17 +10,34 @@ (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages installers) (gnu packages linux) + (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (guix build-system gnu) (guix build-system trivial) (guix gexp) (guix packages) (guix profiles) (guix utils)) +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + (define (make-gcc-rpath-link xgcc) "Given a XGCC package, return a modified package that replace each instance of -rpath in the default system spec that's inserted by Guix with -rpath-link" @@ -102,45 +119,77 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc gcc-9) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + + (packages->manifest - (list ;; The Basics - bash-minimal - which - coreutils - util-linux - ;; File(system) inspection - file - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - ;; Compression and archiving - tar - bzip2 - gzip - xz - zlib - ;; Build tools - gnu-make - libtool - autoconf - automake - pkg-config - ;; Scripting - perl - python-3.7 - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27) - ;; Cross gcc 9 toolchains targeting glibc 2.27 - (make-bitcoin-cross-toolchain "i686-linux-gnu") - (make-bitcoin-cross-toolchain "x86_64-linux-gnu") - (make-bitcoin-cross-toolchain "aarch64-linux-gnu") - (make-bitcoin-cross-toolchain "arm-linux-gnueabihf") - ;; The glibc 2.27 for riscv64 needs gcc 7 to successfully build (see: - ;; https://www.gnu.org/software/gcc/gcc-7/changes.html#riscv). The final - ;; toolchain is still a gcc 9 toolchain targeting glibc 2.27. - (make-bitcoin-cross-toolchain "riscv64-linux-gnu" - #:base-gcc-for-libc gcc-7))) + (append + (list ;; The Basics + bash-minimal + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + ;; Scripting + perl + python-3.7 + ;; Native gcc 9 toolchain targeting glibc 2.27 + (make-gcc-toolchain gcc-9 glibc-2.27)) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) + ((string-contains target "riscv64-linux-") + (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" + #:base-gcc-for-libc gcc-7))) + ((string-contains target "-linux-") + (list (make-bitcoin-cross-toolchain target))) + (else '()))))) diff --git a/doc/release-notes.md b/doc/release-notes.md index cd6a4d6b59..db5aa0d8a7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,5 +1,73 @@ -Please edit the release notes here: +*After branching off for a major version release of Bitcoin Core, use this +template to create the initial release notes draft.* -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/0.20.0-Release-Notes-Draft +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*Create the draft, named* "*version* Release Notes Draft" +*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* + +https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ *Before the final release, move the notes back to this git repository.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-*version*/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From Bitcoin Core 0.17.0 onwards, macOS versions earlier than 10.10 are no +longer supported, as Bitcoin Core is now built using Qt 5.9.x which requires +macOS 10.10+. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +In addition to previously supported CPU platforms, this release's pre-compiled +distribution provides binaries for the RISC-V platform. + +Notable changes +=============== + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/share/setup.nsi.in b/share/setup.nsi.in index dd9ee54d6d..4b2903a7c9 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -2,6 +2,11 @@ Name "@PACKAGE_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma +SetDateSave off + +# Uncomment these lines when investigating reproducibility errors +#SetCompress off +#SetDatablockOptimize off # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" diff --git a/src/Makefile.am b/src/Makefile.am index ae95902f67..f15852ac66 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -523,9 +523,12 @@ libbitcoin_util_a_SOURCES = \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ - util/url.cpp \ $(BITCOIN_CORE_H) +if USE_LIBEVENT +libbitcoin_util_a_SOURCES += util/url.cpp +endif + if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp AM_LDFLAGS += $(COMPAT_LDFLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d53477793c..4def9b4f49 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -245,6 +245,7 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ test/validationinterface_tests.cpp \ test/versionbits_tests.cpp diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 6982eaab61..c27f404a48 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -15,6 +15,7 @@ #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <functional> #include <memory> @@ -29,6 +30,7 @@ #include <compat/stdin.h> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -228,6 +230,7 @@ public: const int ID_NETWORKINFO = 0; const int ID_BLOCKCHAININFO = 1; const int ID_WALLETINFO = 2; + const int ID_BALANCES = 3; /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override @@ -239,6 +242,7 @@ public: result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); result.push_back(JSONRPCRequestObj("getwalletinfo", NullUniValue, ID_WALLETINFO)); + result.push_back(JSONRPCRequestObj("getbalances", NullUniValue, ID_BALANCES)); return result; } @@ -246,9 +250,9 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, 3); - // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on - // getwalletinfo() is allowed to fail in case there is no wallet. + std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; + // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { return batch[ID_NETWORKINFO]; } @@ -265,13 +269,15 @@ public: result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); } result.pushKV("paytxfee", batch[ID_WALLETINFO]["result"]["paytxfee"]); } + if (!batch[ID_BALANCES]["result"].isNull()) { + result.pushKV("balance", batch[ID_BALANCES]["result"]["mine"]["trusted"]); + } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); return JSONRPCReplyObj(result, NullUniValue, 1); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f1a4a114b..76152a81d8 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -11,11 +11,13 @@ #include <logging.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <wallet/wallettool.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs() { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e284dce0d5..f26eb45fce 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -20,10 +20,12 @@ #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> +#include <util/url.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static void WaitForShutdown(NodeContext& node) { diff --git a/src/init.cpp b/src/init.cpp index 437e934093..de2db694fd 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,13 +243,12 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - // - // g_chainstate is referenced here directly (instead of ::ChainstateActive()) because it - // may not have been initialized yet. { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + } } } @@ -273,9 +272,11 @@ void Shutdown(NodeContext& node) { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); - g_chainstate->ResetCoinsViews(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + chainstate->ResetCoinsViews(); + } } pblocktree.reset(); } @@ -291,13 +292,6 @@ void Shutdown(NodeContext& node) } #endif - try { - if (!fs::remove(GetPidFile())) { - LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); - } - } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); - } node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); @@ -305,6 +299,15 @@ void Shutdown(NodeContext& node) ECC_Stop(); if (node.mempool) node.mempool = nullptr; node.scheduler.reset(); + + try { + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } + } catch (const fs::filesystem_error& e) { + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); + } + LogPrintf("%s: done\n", __func__); } @@ -717,11 +720,17 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); - return; + + // We can't hold cs_main during ActivateBestChain even though we're accessing + // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the relevant pointers before the ABC call. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { + LogPrintf("Failed to connect best block (%s)\n", state.ToString()); + StartShutdown(); + return; + } } if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { @@ -1511,17 +1520,18 @@ bool AppInitMain(NodeContext& node) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { bool fReset = fReindex; + auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + }; std::string strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); do { const int64_t load_block_index_start_time = GetTimeMillis(); - bool is_coinsview_empty; try { LOCK(cs_main); - // This statement makes ::ChainstateActive() usable. - g_chainstate = MakeUnique<CChainState>(); + g_chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1574,43 +1584,53 @@ bool AppInitMain(NodeContext& node) // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! - ::ChainstateActive().InitCoinsDB( - /* cache_size_bytes */ nCoinDBCache, - /* in_memory */ false, - /* should_wipe */ fReset || fReindexChainState); - - ::ChainstateActive().CoinsErrorCatcher().AddReadErrCallback([]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, - "", CClientUIInterface::MSG_ERROR); - }); - - // If necessary, upgrade from older database format. - // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; - break; - } - - // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; - break; - } - - // The on-disk coinsdb is now in a good state, create the cache - ::ChainstateActive().InitCoinsCache(); - assert(::ChainstateActive().CanFlushToDisk()); + bool failed_chainstate_init = false; + + for (CChainState* chainstate : g_chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( + /* cache_size_bytes */ nCoinDBCache, + /* in_memory */ false, + /* should_wipe */ fReset || fReindexChainState); + + chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down.").translated, + "", CClientUIInterface::MSG_ERROR); + }); + + // If necessary, upgrade from older database format. + // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->CoinsDB().Upgrade()) { + strLoadError = _("Error upgrading chainstate database").translated; + failed_chainstate_init = true; + break; + } - is_coinsview_empty = fReset || fReindexChainState || - ::ChainstateActive().CoinsTip().GetBestBlock().IsNull(); - if (!is_coinsview_empty) { - // LoadChainTip initializes the chain based on CoinsTip()'s best block - if (!::ChainstateActive().LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->ReplayBlocks(chainparams)) { + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + failed_chainstate_init = true; break; } - assert(::ChainActive().Tip() != nullptr); + + // The on-disk coinsdb is now in a good state, create the cache + chainstate->InitCoinsCache(); + assert(chainstate->CanFlushToDisk()); + + if (!is_coinsview_empty(chainstate)) { + // LoadChainTip initializes the chain based on CoinsTip()'s best block + if (!chainstate->LoadChainTip(chainparams)) { + strLoadError = _("Error initializing block database").translated; + failed_chainstate_init = true; + break; // out of the per-chainstate loop + } + assert(chainstate->m_chain.Tip() != nullptr); + } + } + + if (failed_chainstate_init) { + break; // out of the chainstate activation do-while } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); @@ -1618,49 +1638,76 @@ bool AppInitMain(NodeContext& node) break; } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on ::ChainActive(), and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain").translated; - break; + bool failed_rewind{false}; + // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant + // chainstates beforehand. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on the chainstate, and drops block data in + // BlockIndex() based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...").translated); + if (!chainstate->RewindBlockIndex(chainparams)) { + strLoadError = _( + "Unable to rewind the database to a pre-fork state. " + "You will need to redownload the blockchain").translated; + failed_rewind = true; + break; // out of the per-chainstate loop + } } } + if (failed_rewind) { + break; // out of the chainstate activation do-while + } + + bool failed_verification = false; + try { LOCK(cs_main); - if (!is_coinsview_empty) { - uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { - LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - CBlockIndex* tip = ::ChainActive().Tip(); - RPCNotifyBlockChange(true, tip); - if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; - break; - } - - if (!CVerifyDB().VerifyDB(chainparams, &::ChainstateActive().CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; - break; + for (CChainState* chainstate : g_chainman.GetAll()) { + if (!is_coinsview_empty(chainstate)) { + uiInterface.InitMessage(_("Verifying blocks...").translated); + if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + + const CBlockIndex* tip = chainstate->m_chain.Tip(); + RPCNotifyBlockChange(true, tip); + if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { + strLoadError = _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + failed_verification = true; + break; + } + + // Only verify the DB of the active chainstate. This is fixed in later + // work when we allow VerifyDB to be parameterized by chainstate. + if (&::ChainstateActive() == chainstate && + !CVerifyDB().VerifyDB( + chainparams, &chainstate->CoinsDB(), + gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + strLoadError = _("Corrupted block database detected").translated; + failed_verification = true; + break; + } } } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database").translated; + failed_verification = true; break; } - fLoaded = true; - LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + if (!failed_verification) { + fLoaded = true; + LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + } } while(false); if (!fLoaded && !ShutdownRequested()) { @@ -1724,8 +1771,11 @@ bool AppInitMain(NodeContext& node) LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { - uiInterface.InitMessage(_("Pruning blockstore...").translated); - ::ChainstateActive().PruneAndFlush(); + LOCK(cs_main); + for (CChainState* chainstate : g_chainman.GetAll()) { + uiInterface.InitMessage(_("Pruning blockstore...").translated); + chainstate->PruneAndFlush(); + } } } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index e1bc9bbbf3..ffa9e90c79 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -283,6 +283,12 @@ public: //! Shut down client. virtual void stop() = 0; + + //! Set mock time. + virtual void setMockTime(int64_t time) = 0; + + //! Return interfaces for accessing wallets (if any). + virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 905173d20b..6e5fdc61b5 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -253,8 +253,9 @@ public: std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; - for (const std::shared_ptr<CWallet>& wallet : GetWallets()) { - wallets.emplace_back(MakeWallet(wallet)); + for (auto& client : m_context.chain_clients) { + auto client_wallets = client->getWallets(); + std::move(client_wallets.begin(), client_wallets.end(), std::back_inserter(wallets)); } return wallets; } diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index abce09ca4a..c6e0db34f7 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -519,6 +519,15 @@ public: void start(CScheduler& scheduler) override { return StartWallets(scheduler); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } + void setMockTime(int64_t time) override { return SetMockTime(time); } + std::vector<std::unique_ptr<Wallet>> getWallets() override + { + std::vector<std::unique_ptr<Wallet>> wallets; + for (const auto& wallet : GetWallets()) { + wallets.emplace_back(MakeWallet(wallet)); + } + return wallets; + } ~WalletClientImpl() override { UnloadWallets(); } Chain& m_chain; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index dbb0912230..df2ab89d3f 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -15,7 +15,6 @@ #include <functional> #include <map> #include <memory> -#include <psbt.h> #include <stdint.h> #include <string> #include <tuple> @@ -26,12 +25,13 @@ class CCoinControl; class CFeeRate; class CKey; class CWallet; -enum isminetype : unsigned int; enum class FeeReason; -typedef uint8_t isminefilter; - enum class OutputType; +enum class TransactionError; +enum isminetype : unsigned int; struct CRecipient; +struct PartiallySignedTransaction; +typedef uint8_t isminefilter; namespace interfaces { diff --git a/src/miner.cpp b/src/miner.cpp index 61d27d17c1..5ab23c7f4f 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -39,6 +39,17 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } +void RegenerateCommitments(CBlock& block) +{ + CMutableTransaction tx{*block.vtx.at(0)}; + tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); + block.vtx.at(0) = MakeTransactionRef(tx); + + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + + block.hashMerkleRoot = BlockMerkleRoot(block); +} + BlockAssembler::Options::Options() { blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; diff --git a/src/miner.h b/src/miner.h index cc8fc31a9f..b9ffb34a2d 100644 --- a/src/miner.h +++ b/src/miner.h @@ -203,4 +203,7 @@ private: void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ +void RegenerateCommitments(CBlock& block); + #endif // BITCOIN_MINER_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 4313d6ee7f..0ec1687dc5 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -336,7 +336,7 @@ void BitcoinApplication::initializeResult(bool success) window->setClientModel(clientModel); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { - m_wallet_controller = new WalletController(m_node, platformStyle, optionsModel, this); + m_wallet_controller = new WalletController(*clientModel, platformStyle, this); window->setWalletController(m_wallet_controller); if (paymentServer) { paymentServer->setOptionsModel(optionsModel); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a1ec3eaab1..2a69876b6d 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -105,6 +105,14 @@ int64_t ClientModel::getHeaderTipTime() const return cachedBestHeaderTime; } +int ClientModel::getNumBlocks() const +{ + if (m_cached_num_blocks == -1) { + m_cached_num_blocks = m_node.getNumBlocks(); + } + return m_cached_num_blocks; +} + void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); @@ -241,6 +249,8 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int heig // cache best headers time and height to reduce future cs_main locks clientmodel->cachedBestHeaderHeight = height; clientmodel->cachedBestHeaderTime = blockTime; + } else { + clientmodel->m_cached_num_blocks = height; } // During initial sync, block notifications, and header notifications from reindexing are both throttled. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 79175e0af4..7ac4120a8f 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -56,6 +56,7 @@ public: //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; + int getNumBlocks() const; int getHeaderTipHeight() const; int64_t getHeaderTipTime() const; @@ -73,9 +74,10 @@ public: bool getProxyInfo(std::string& ip_port) const; - // caches for the best header + // caches for the best header, number of blocks mutable std::atomic<int> cachedBestHeaderHeight; mutable std::atomic<int64_t> cachedBestHeaderTime; + mutable std::atomic<int> m_cached_num_blocks{-1}; private: interfaces::Node& m_node; diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 3dfd9e850e..607cf9f976 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -5,6 +5,7 @@ #include <qt/bitcoin.h> #include <util/translation.h> +#include <util/url.h> #include <QCoreApplication> @@ -15,5 +16,6 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; +UrlDecodeFn* const URL_DECODE = urlDecode; int main(int argc, char* argv[]) { return GuiMain(argc, argv); } diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 042387286a..a3bf56dc07 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <qt/clientmodel.h> #include <qt/editaddressdialog.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> @@ -106,8 +107,9 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 14a75b23f3..064b9ceb18 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -82,6 +82,7 @@ void AppTests::appTests() // Reset global state to avoid interfering with later tests. AbortShutdown(); UnloadBlockIndex(); + WITH_LOCK(::cs_main, g_chainman.Reset()); } //! Entry point for BitcoinGUI tests. diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index c1a0f63f73..41f377d6d2 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> #include <qt/bitcoinamountfield.h> +#include <qt/clientmodel.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/qvalidatedlineedit.h> @@ -168,8 +169,9 @@ void TestGUI(interfaces::Node& node) SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 64e9c856db..18554aef1f 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -5,6 +5,7 @@ #include <qt/transactiontablemodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -175,7 +176,7 @@ public: return cachedWallet.size(); } - TransactionRecord *index(interfaces::Wallet& wallet, int idx) + TransactionRecord *index(interfaces::Wallet& wallet, const int cur_num_blocks, const int idx) { if(idx >= 0 && idx < cachedWallet.size()) { @@ -191,7 +192,7 @@ public: interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time) && rec->statusUpdateNeeded(numBlocks)) { + if (rec->statusUpdateNeeded(cur_num_blocks) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, numBlocks, block_time); } return rec; @@ -663,10 +664,10 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), row); + TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); if(data) { - return createIndex(row, column, priv->index(walletModel->wallet(), row)); + return createIndex(row, column, data); } return QModelIndex(); } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index f076b5ba61..7d5c33ad02 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -5,6 +5,7 @@ #include <qt/walletcontroller.h> #include <qt/askpassphrasedialog.h> +#include <qt/clientmodel.h> #include <qt/createwalletdialog.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -24,13 +25,14 @@ #include <QTimer> #include <QWindow> -WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) +WalletController::WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent) : QObject(parent) , m_activity_thread(new QThread(this)) , m_activity_worker(new QObject) - , m_node(node) + , m_client_model(client_model) + , m_node(client_model.node()) , m_platform_style(platform_style) - , m_options_model(options_model) + , m_options_model(client_model.getOptionsModel()) { m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { getOrCreateWallet(std::move(wallet)); @@ -104,7 +106,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal } // Instantiate model and register it. - WalletModel* wallet_model = new WalletModel(std::move(wallet), m_node, m_platform_style, m_options_model, nullptr); + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, nullptr); // Handler callback runs in a different thread so fix wallet model thread affinity. wallet_model->moveToThread(thread()); wallet_model->setParent(this); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f30eb25308..5840c3343f 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -21,6 +21,7 @@ #include <QTimer> #include <QString> +class ClientModel; class OptionsModel; class PlatformStyle; class WalletModel; @@ -47,7 +48,7 @@ class WalletController : public QObject void removeAndDeleteWallet(WalletModel* wallet_model); public: - WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent); + WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); //! Returns wallet models currently open. @@ -70,6 +71,7 @@ Q_SIGNALS: private: QThread* const m_activity_thread; QObject* const m_activity_worker; + ClientModel& m_client_model; interfaces::Node& m_node; const PlatformStyle* const m_platform_style; OptionsModel* const m_options_model; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 608797d6ad..9febebf905 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -9,6 +9,7 @@ #include <qt/walletmodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -20,6 +21,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <wallet/coincontrol.h> @@ -33,8 +35,13 @@ #include <QTimer> -WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *_optionsModel, QObject *parent) : - QObject(parent), m_wallet(std::move(wallet)), m_node(node), optionsModel(_optionsModel), addressTableModel(nullptr), +WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : + QObject(parent), + m_wallet(std::move(wallet)), + m_client_model(client_model), + m_node(client_model.node()), + optionsModel(client_model.getOptionsModel()), + addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 7936014af9..643cfc7fb9 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -24,6 +24,7 @@ enum class OutputType; class AddressTableModel; +class ClientModel; class OptionsModel; class PlatformStyle; class RecentRequestsTableModel; @@ -52,7 +53,7 @@ class WalletModel : public QObject Q_OBJECT public: - explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *optionsModel, QObject *parent = nullptr); + explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent = nullptr); ~WalletModel(); enum StatusCode // Returned by sendCoins @@ -143,6 +144,7 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } + ClientModel& clientModel() const { return m_client_model; } QString getWalletName() const; QString getDisplayName() const; @@ -159,6 +161,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; + ClientModel& m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f71ed99652..07171ab8c4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1078,13 +1078,11 @@ UniValue gettxout(const JSONRPCRequest& request) static UniValue verifychain(const JSONRPCRequest& request) { - int nCheckLevel = gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL); - int nCheckDepth = gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, - {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), "How thorough the block verification is."}, + {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS), "The number of blocks to check."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Verified or not"}, @@ -1094,15 +1092,12 @@ static UniValue verifychain(const JSONRPCRequest& request) }, }.Check(request); - LOCK(cs_main); + const int check_level(request.params[0].isNull() ? DEFAULT_CHECKLEVEL : request.params[0].get_int()); + const int check_depth{request.params[1].isNull() ? DEFAULT_CHECKBLOCKS : request.params[1].get_int()}; - if (!request.params[0].isNull()) - nCheckLevel = request.params[0].get_int(); - if (!request.params[1].isNull()) - nCheckDepth = request.params[1].get_int(); + LOCK(cs_main); - return CVerifyDB().VerifyDB( - Params(), &::ChainstateActive().CoinsTip(), nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), &::ChainstateActive().CoinsTip(), check_level, check_depth); } static void BuriedForkDescPushBack(UniValue& softforks, const std::string &name, int height) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -1319,7 +1314,7 @@ static UniValue getchaintips(const JSONRPCRequest& request) /* * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: - * - Make one pass through g_blockman.m_block_index, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. + * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. * - add ::ChainActive().Tip() */ diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c1762483e9..84719d4ca4 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -33,6 +33,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "generatetoaddress", 2, "maxtries" }, { "generatetodescriptor", 0, "num_blocks" }, { "generatetodescriptor", 2, "maxtries" }, + { "generateblock", 1, "transactions" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, @@ -97,10 +98,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 1, "privkeys" }, { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, - { "sendrawtransaction", 1, "allowhighfees" }, { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, - { "testmempoolaccept", 1, "allowhighfees" }, { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index da9d583fa7..b812f3005f 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,6 +101,36 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } +static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +{ + block_hash.SetNull(); + + { + LOCK(cs_main); + IncrementExtraNonce(&block, ::ChainActive().Tip(), extra_nonce); + } + + CChainParams chainparams(Params()); + + while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus()) && !ShutdownRequested()) { + ++block.nNonce; + --max_tries; + } + if (max_tries == 0 || ShutdownRequested()) { + return false; + } + if (block.nNonce == std::numeric_limits<uint32_t>::max()) { + return true; + } + + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + + block_hash = block.GetHash(); + return true; +} + static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; @@ -119,29 +149,54 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; - { - LOCK(cs_main); - IncrementExtraNonce(pblock, ::ChainActive().Tip(), nExtraNonce); - } - while (nMaxTries > 0 && pblock->nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(pblock->GetHash(), pblock->nBits, Params().GetConsensus()) && !ShutdownRequested()) { - ++pblock->nNonce; - --nMaxTries; - } - if (nMaxTries == 0 || ShutdownRequested()) { + + uint256 block_hash; + if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { break; } - if (pblock->nNonce == std::numeric_limits<uint32_t>::max()) { - continue; + + if (!block_hash.IsNull()) { + ++nHeight; + blockHashes.push_back(block_hash.GetHex()); } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr)) - throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); - ++nHeight; - blockHashes.push_back(pblock->GetHash().GetHex()); } return blockHashes; } +static bool getScriptFromDescriptor(const std::string& descriptor, CScript& script, std::string& error) +{ + FlatSigningProvider key_provider; + const auto desc = Parse(descriptor, key_provider, error, /* require_checksum = */ false); + if (desc) { + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> scripts; + if (!desc->Expand(0, key_provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + // Combo desriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 + CHECK_NONFATAL(scripts.size() > 0 && scripts.size() <= 4); + + if (scripts.size() == 1) { + script = scripts.at(0); + } else if (scripts.size() == 4) { + // For uncompressed keys, take the 3rd script, since it is p2wpkh + script = scripts.at(2); + } else { + // Else take the 2nd script, since it is p2pkh + script = scripts.at(1); + } + + return true; + } else { + return false; + } +} + static UniValue generatetodescriptor(const JSONRPCRequest& request) { RPCHelpMan{ @@ -166,27 +221,15 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) const int num_blocks{request.params[0].get_int()}; const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; - FlatSigningProvider key_provider; + CScript coinbase_script; std::string error; - const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); - if (!desc) { + if (!getScriptFromDescriptor(request.params[1].get_str(), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - if (desc->IsRange()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); - } - - FlatSigningProvider provider; - std::vector<CScript> coinbase_script; - if (!desc->Expand(0, key_provider, coinbase_script, provider)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); - } const CTxMemPool& mempool = EnsureMemPool(); - CHECK_NONFATAL(coinbase_script.size() == 1); - - return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); + return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -229,6 +272,113 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } +static UniValue generateblock(const JSONRPCRequest& request) +{ + RPCHelpMan{"generateblock", + "\nMine a block with a set of ordered transactions immediately to a specified address or descriptor (before the RPC call returns)\n", + { + {"address/descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The address or descriptor to send the newly generated bitcoin to."}, + {"transactions", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings which are either txids or raw transactions.\n" + "Txids must reference transactions currently in the mempool.\n" + "All transactions must be valid and in valid order, otherwise the block will be rejected.", + { + {"rawtx/txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + } + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "hash", "hash of generated block"} + } + }, + RPCExamples{ + "\nGenerate a block to myaddress, with txs rawtx and mempool_txid\n" + + HelpExampleCli("generateblock", R"("myaddress" '["rawtx", "mempool_txid"]')") + }, + }.Check(request); + + const auto address_or_descriptor = request.params[0].get_str(); + CScript coinbase_script; + std::string error; + + if (!getScriptFromDescriptor(address_or_descriptor, coinbase_script, error)) { + const auto destination = DecodeDestination(address_or_descriptor); + if (!IsValidDestination(destination)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address or descriptor"); + } + + coinbase_script = GetScriptForDestination(destination); + } + + const CTxMemPool& mempool = EnsureMemPool(); + + std::vector<CTransactionRef> txs; + const auto raw_txs_or_txids = request.params[1].get_array(); + for (size_t i = 0; i < raw_txs_or_txids.size(); i++) { + const auto str(raw_txs_or_txids[i].get_str()); + + uint256 hash; + CMutableTransaction mtx; + if (ParseHashStr(str, hash)) { + + const auto tx = mempool.get(hash); + if (!tx) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Transaction %s not in mempool.", str)); + } + + txs.emplace_back(tx); + + } else if (DecodeHexTx(mtx, str)) { + txs.push_back(MakeTransactionRef(std::move(mtx))); + + } else { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Transaction decode failed for %s", str)); + } + } + + CChainParams chainparams(Params()); + CBlock block; + + { + LOCK(cs_main); + + CTxMemPool empty_mempool; + std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(empty_mempool, chainparams).CreateNewBlock(coinbase_script)); + if (!blocktemplate) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + } + block = blocktemplate->block; + } + + CHECK_NONFATAL(block.vtx.size() == 1); + + // Add transactions + block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); + RegenerateCommitments(block); + + { + LOCK(cs_main); + + BlockValidationState state; + if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); + } + } + + uint256 block_hash; + uint64_t max_tries{1000000}; + unsigned int extra_nonce{0}; + + if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); + } + + UniValue obj(UniValue::VOBJ); + obj.pushKV("hash", block_hash.GetHex()); + return obj; +} + static UniValue getmininginfo(const JSONRPCRequest& request) { RPCHelpMan{"getmininginfo", @@ -1038,6 +1188,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, + { "generating", "generateblock", &generateblock, {"address","transactions"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 51a9581349..0525bec6fd 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -363,7 +364,13 @@ static UniValue setmocktime(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - SetMockTime(request.params[0].get_int64()); + int64_t time = request.params[0].get_int64(); + SetMockTime(time); + if (g_rpc_node) { + for (const auto& chain_client : g_rpc_node->chain_clients) { + chain_client->setMockTime(time); + } + } return NullUniValue; } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 3c0b060439..e86813ef03 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -825,7 +825,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VSTR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); // parse hex string from parameter @@ -834,13 +834,9 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -896,7 +892,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VARR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); if (request.params[0].get_array().size() != 1) { @@ -910,13 +906,9 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); CTxMemPool& mempool = EnsureMemPool(); int64_t virtual_size = GetVirtualTransactionSize(*tx); @@ -1823,10 +1815,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 083022fbdd..50f7806ba5 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -342,6 +342,35 @@ public: }; } +/** Helper for OP_CHECKSIG and OP_CHECKSIGVERIFY + * + * A return value of false means the script fails entirely. When true is returned, the + * fSuccess variable indicates whether the signature check itself succeeded. + */ +static bool EvalChecksig(const valtype& vchSig, const valtype& vchPubKey, CScript::const_iterator pbegincodehash, CScript::const_iterator pend, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror, bool& fSuccess) +{ + // Subset of script starting at the most recent codeseparator + CScript scriptCode(pbegincodehash, pend); + + // Drop the signature in pre-segwit scripts but not segwit scripts + if (sigversion == SigVersion::BASE) { + int found = FindAndDelete(scriptCode, CScript() << vchSig); + if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) + return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); + } + + if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { + //serror is set + return false; + } + fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); + + if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) + return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); + + return true; +} + bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) { static const CScriptNum bnZero(0); @@ -985,25 +1014,8 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& valtype& vchSig = stacktop(-2); valtype& vchPubKey = stacktop(-1); - // Subset of script starting at the most recent codeseparator - CScript scriptCode(pbegincodehash, pend); - - // Drop the signature in pre-segwit scripts but not segwit scripts - if (sigversion == SigVersion::BASE) { - int found = FindAndDelete(scriptCode, CScript() << vchSig); - if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) - return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); - } - - if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { - //serror is set - return false; - } - bool fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); - - if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) - return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); - + bool fSuccess = true; + if (!EvalChecksig(vchSig, vchPubKey, pbegincodehash, pend, flags, checker, sigversion, serror, fSuccess)) return false; popstack(stack); popstack(stack); stack.push_back(fSuccess ? vchTrue : vchFalse); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index a4d0126925..32d50e49b9 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -27,12 +27,14 @@ #include <util/string.h> #include <util/time.h> #include <util/translation.h> +#include <util/url.h> #include <validation.h> #include <validationinterface.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; FastRandomContext g_insecure_rand_ctx; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ @@ -112,7 +114,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainstate = MakeUnique<CChainState>(); + + g_chainman.InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -159,7 +162,7 @@ TestingSetup::~TestingSetup() m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainstate.reset(); + g_chainman.Reset(); pblocktree.reset(); } diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp new file mode 100644 index 0000000000..6e7186db22 --- /dev/null +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -0,0 +1,104 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <chainparams.h> +#include <random.h> +#include <uint256.h> +#include <consensus/validation.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) + +//! Basic tests for ChainstateManager. +//! +//! First create a legacy (IBD) chainstate, then create a snapshot chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager) +{ + ChainstateManager manager; + std::vector<CChainState*> chainstates; + const CChainParams& chainparams = Params(); + + // Create a legacy (IBD) chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache()); + + BOOST_CHECK(!manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); + auto all = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), -1); + + auto active_tip = manager.ActiveTip(); + auto exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip, exp_tip); + + auto& validated_cs = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs, &c1); + + // Create a snapshot-based chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c2.InitCoinsCache()); + // Unlike c1, which doesn't have any blocks. Gets us different tip, height. + c2.LoadGenesisBlock(chainparams); + BlockValidationState _; + BOOST_CHECK(c2.ActivateBestChain(_, chainparams, nullptr)); + + BOOST_CHECK(manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(manager.IsBackgroundIBD(&c1)); + BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + auto all2 = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain2 = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), 0); + + auto active_tip2 = manager.ActiveTip(); + auto exp_tip2 = c2.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip2, exp_tip2); + + // Ensure that these pointers actually correspond to different + // CCoinsViewCache instances. + BOOST_CHECK(exp_tip != exp_tip2); + + auto& validated_cs2 = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs2, &c1); + + auto& validated_chain = manager.ValidatedChain(); + BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); + + auto validated_tip = manager.ValidatedTip(); + exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(validated_tip, exp_tip); + + // Avoid triggering the address sanitizer. + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/url.h b/src/util/url.h index e9ea2ab765..be9f1c9e8a 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -7,6 +7,8 @@ #include <string> -std::string urlDecode(const std::string &urlEncoded); +using UrlDecodeFn = std::string(const std::string& url_encoded); +UrlDecodeFn urlDecode; +extern UrlDecodeFn* const URL_DECODE; #endif // BITCOIN_UTIL_URL_H diff --git a/src/validation.cpp b/src/validation.cpp index 7ee94f8657..9f5c59e52b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <optional.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -76,20 +77,19 @@ bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIn return false; } -namespace { -BlockManager g_blockman; -} // anon namespace - -std::unique_ptr<CChainState> g_chainstate; +ChainstateManager g_chainman; -CChainState& ChainstateActive() { - assert(g_chainstate); - return *g_chainstate; +CChainState& ChainstateActive() +{ + LOCK(::cs_main); + assert(g_chainman.m_active_chainstate); + return *g_chainman.m_active_chainstate; } -CChain& ChainActive() { - assert(g_chainstate); - return g_chainstate->m_chain; +CChain& ChainActive() +{ + LOCK(::cs_main); + return ::ChainstateActive().m_chain; } /** @@ -151,8 +151,8 @@ namespace { CBlockIndex* LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_blockman.m_block_index.find(hash); - return it == g_blockman.m_block_index.end() ? nullptr : it->second; + BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); + return it == g_chainman.BlockIndex().end() ? nullptr : it->second; } CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) @@ -1242,10 +1242,9 @@ void CoinsViews::InitCache() m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); } -// NOTE: for now m_blockman is set to a global, but this will be changed -// in a future commit. -CChainState::CChainState() : m_blockman(g_blockman) {} - +CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) + : m_blockman(blockman), + m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( size_t cache_size_bytes, @@ -1253,6 +1252,10 @@ void CChainState::InitCoinsDB( bool should_wipe, std::string leveldb_name) { + if (!m_from_snapshot_blockhash.IsNull()) { + leveldb_name += "_" + m_from_snapshot_blockhash.ToString(); + } + m_coins_views = MakeUnique<CoinsViews>( leveldb_name, cache_size_bytes, in_memory, should_wipe); } @@ -1294,7 +1297,8 @@ static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; BlockMap& BlockIndex() { - return g_blockman.m_block_index; + LOCK(::cs_main); + return g_chainman.m_blockman.m_block_index; } static void AlertNotify(const std::string& strMessage) @@ -3443,7 +3447,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio if (fCheckpointsEnabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our - // g_blockman.m_block_index. + // BlockIndex(). CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); @@ -3651,7 +3655,8 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValid LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_blockman.AcceptBlockHeader(header, state, chainparams, &pindex); + bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { @@ -3853,7 +3858,7 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (const auto& entry : g_blockman.m_block_index) { + for (const auto& entry : g_chainman.BlockIndex()) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3867,12 +3872,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_blockman.m_blocks_unlinked.erase(_it); + g_chainman.m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -4109,9 +4114,11 @@ void BlockManager::Unload() { bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_blockman.LoadBlockIndex( - chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) + if (!g_chainman.m_blockman.LoadBlockIndex( + chainparams.GetConsensus(), *pblocktree, + ::ChainstateActive().setBlockIndexCandidates)) { return false; + } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); @@ -4133,7 +4140,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_blockman.m_block_index) + for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4510,26 +4517,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); - } - } - - return true; -} -bool RewindBlockIndex(const CChainParams& params) { - if (!::ChainstateActive().RewindBlockIndex(params)) { - return false; - } - - LOCK(cs_main); - if (::ChainActive().Tip() != nullptr) { - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; + // FlushStateToDisk can possibly read ::ChainActive(). Be conservative + // and skip it here, we're about to -reindex-chainstate anyway, so + // it'll get called a bunch real soon. + BlockValidationState state; + if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } } } @@ -4547,8 +4543,7 @@ void CChainState::UnloadBlockIndex() { void UnloadBlockIndex() { LOCK(cs_main); - ::ChainActive().SetTip(nullptr); - g_blockman.Unload(); + g_chainman.Unload(); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; mempool.clear(); @@ -4561,8 +4556,6 @@ void UnloadBlockIndex() warningcache[b].clear(); } fHavePruned = false; - - ::ChainstateActive().UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -4572,7 +4565,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) if (!fReindex) { bool ret = LoadBlockIndexDB(chainparams); if (!ret) return false; - needs_init = g_blockman.m_block_index.empty(); + needs_init = g_chainman.m_blockman.m_block_index.empty(); } if (needs_init) { @@ -4693,7 +4686,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } @@ -4923,6 +4916,14 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } +std::string CChainState::ToString() +{ + CBlockIndex* tip = m_chain.Tip(); + return strprintf("Chainstate [%s] @ height %d (%s)", + m_from_snapshot_blockhash.IsNull() ? "ibd" : "snapshot", + tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); @@ -5110,10 +5111,99 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - BlockMap::iterator it1 = g_blockman.m_block_index.begin(); - for (; it1 != g_blockman.m_block_index.end(); it1++) + BlockMap::iterator it1 = g_chainman.BlockIndex().begin(); + for (; it1 != g_chainman.BlockIndex().end(); it1++) delete (*it1).second; - g_blockman.m_block_index.clear(); + g_chainman.BlockIndex().clear(); } }; static CMainCleanup instance_of_cmaincleanup; + +Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + if (m_active_chainstate != nullptr) { + // If a snapshot chainstate exists, it will always be our active. + return m_active_chainstate->m_from_snapshot_blockhash; + } + return {}; +} + +std::vector<CChainState*> ChainstateManager::GetAll() +{ + std::vector<CChainState*> out; + + if (!IsSnapshotValidated() && m_ibd_chainstate) { + out.push_back(m_ibd_chainstate.get()); + } + + if (m_snapshot_chainstate) { + out.push_back(m_snapshot_chainstate.get()); + } + + return out; +} + +CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +{ + bool is_snapshot = !snapshot_blockhash.IsNull(); + std::unique_ptr<CChainState>& to_modify = + is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + + if (to_modify) { + throw std::logic_error("should not be overwriting a chainstate"); + } + + to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + + // Snapshot chainstates and initial IBD chaintates always become active. + if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { + LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); + m_active_chainstate = to_modify.get(); + } else { + throw std::logic_error("unexpected chainstate activation"); + } + + return *to_modify; +} + +CChain& ChainstateManager::ActiveChain() const +{ + assert(m_active_chainstate); + return m_active_chainstate->m_chain; +} + +bool ChainstateManager::IsSnapshotActive() const +{ + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); +} + +CChainState& ChainstateManager::ValidatedChainstate() const +{ + if (m_snapshot_chainstate && IsSnapshotValidated()) { + return *m_snapshot_chainstate.get(); + } + assert(m_ibd_chainstate); + return *m_ibd_chainstate.get(); +} + +bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const +{ + return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); +} + +void ChainstateManager::Unload() +{ + for (CChainState* chainstate : this->GetAll()) { + chainstate->m_chain.SetTip(nullptr); + chainstate->UnloadBlockIndex(); + } + + m_blockman.Unload(); +} + +void ChainstateManager::Reset() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; + m_snapshot_validated = false; +} diff --git a/src/validation.h b/src/validation.h index a5335edc43..dbf7aa28db 100644 --- a/src/validation.h +++ b/src/validation.h @@ -14,6 +14,7 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> @@ -379,9 +380,6 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** When there are blocks in the active chain with missing data, rewind the chainstate and remove them from the block index */ -bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); - /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ int GetWitnessCommitmentIndex(const CBlock& block); @@ -539,6 +537,9 @@ enum class CoinsCacheSizeState OK = 0 }; +// Defined below, but needed for `friend` usage in CChainState. +class ChainstateManager; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -591,8 +592,7 @@ private: std::unique_ptr<CoinsViews> m_coins_views; public: - CChainState(BlockManager& blockman) : m_blockman(blockman) {} - CChainState(); + explicit CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -621,6 +621,13 @@ public: CChain m_chain; /** + * The blockhash which is the base of the snapshot this chainstate was created from. + * + * IsNull() if this chainstate was not created from a snapshot. + */ + const uint256 m_from_snapshot_blockhash{}; + + /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be * missing the data for the block. @@ -741,6 +748,8 @@ public: size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -753,6 +762,8 @@ private: //! Mark a block as not having block data void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + friend ChainstateManager; }; /** Mark a block as precious and reorganize. @@ -768,6 +779,150 @@ bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparam /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** + * Provides an interface for creating and interacting with one or two + * chainstates: an IBD chainstate generated by downloading blocks, and + * an optional snapshot chainstate loaded from a UTXO snapshot. Managed + * chainstates can be maintained at different heights simultaneously. + * + * This class provides abstractions that allow the retrieval of the current + * most-work chainstate ("Active") as well as chainstates which may be in + * background use to validate UTXO snapshots. + * + * Definitions: + * + * *IBD chainstate*: a chainstate whose current state has been "fully" + * validated by the initial block download process. + * + * *Snapshot chainstate*: a chainstate populated by loading in an + * assumeutxo UTXO snapshot. + * + * *Active chainstate*: the chainstate containing the current most-work + * chain. Consulted by most parts of the system (net_processing, + * wallet) as a reflection of the current chain and UTXO set. + * This may either be an IBD chainstate or a snapshot chainstate. + * + * *Background IBD chainstate*: an IBD chainstate for which the + * IBD process is happening in the background while use of the + * active (snapshot) chainstate allows the rest of the system to function. + * + * *Validated chainstate*: the most-work chainstate which has been validated + * locally via initial block download. This will be the snapshot chainstate + * if a snapshot was loaded and all blocks up to the snapshot starting point + * have been downloaded and validated (via background validation), otherwise + * it will be the IBD chainstate. + */ +class ChainstateManager +{ +private: + //! The chainstate used under normal operation (i.e. "regular" IBD) or, if + //! a snapshot is in use, for background validation. + //! + //! Its contents (including on-disk data) will be deleted *upon shutdown* + //! after background validation of the snapshot has completed. We do not + //! free the chainstate contents immediately after it finishes validation + //! to cautiously avoid a case where some other part of the system is still + //! using this pointer (e.g. net_processing). + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_ibd_chainstate; + + //! A chainstate initialized on the basis of a UTXO snapshot. If this is + //! non-null, it is always our active chainstate. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_snapshot_chainstate; + + //! Points to either the ibd or snapshot chainstate; indicates our + //! most-work chain. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + CChainState* m_active_chainstate{nullptr}; + + //! If true, the assumed-valid chainstate has been fully validated + //! by the background validation chainstate. + bool m_snapshot_validated{false}; + + // For access to m_active_chainstate. + friend CChainState& ChainstateActive(); + friend CChain& ChainActive(); + +public: + //! A single BlockManager instance is shared across each constructed + //! chainstate to avoid duplicating block metadata. + BlockManager m_blockman GUARDED_BY(::cs_main); + + //! Instantiate a new chainstate and assign it based upon whether it is + //! from a snapshot. + //! + //! @param[in] snapshot_blockhash If given, signify that this chainstate + //! is based on a snapshot. + CChainState& InitializeChainstate(const uint256& snapshot_blockhash = uint256()) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Get all chainstates currently being used. + std::vector<CChainState*> GetAll(); + + //! The most-work chain. + CChain& ActiveChain() const; + int ActiveHeight() const { return ActiveChain().Height(); } + CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } + + BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + return m_blockman.m_block_index; + } + + bool IsSnapshotActive() const; + + Optional<uint256> SnapshotBlockhash() const; + + //! Is there a snapshot in use and has it been fully validated? + bool IsSnapshotValidated() const { return m_snapshot_validated; } + + //! @returns true if this chainstate is being used to validate an active + //! snapshot in the background. + bool IsBackgroundIBD(CChainState* chainstate) const; + + //! Return the most-work chainstate that has been fully validated. + //! + //! During background validation of a snapshot, this is the IBD chain. After + //! background validation has completed, this is the snapshot chain. + CChainState& ValidatedChainstate() const; + + CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } + CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + + //! Unload block index and chain data before shutdown. + void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Clear (deconstruct) chainstate data. + void Reset(); +}; + +extern ChainstateManager g_chainman GUARDED_BY(::cs_main); + /** @returns the most-work valid chainstate. */ CChainState& ChainstateActive(); @@ -777,11 +932,6 @@ CChain& ChainActive(); /** @returns the global block index map. */ BlockMap& BlockIndex(); -// Most often ::ChainstateActive() should be used instead of this, but some code -// may not be able to assume that this has been initialized yet and so must use it -// directly, e.g. init.cpp. -extern std::unique_ptr<CChainState> g_chainstate; - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 61ad2f1198..cee587aeb4 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -78,9 +78,9 @@ bool HaveKey(const SigningProvider& wallet, const CKey& key) bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { // wallet endpoint was used - wallet_name = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a487e9e2e0..13e10c1eab 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -454,7 +454,7 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK(wallet->cs_wallet); + LOCK2(::cs_main, wallet->cs_wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index f04a58cd19..2e1e84b707 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -6,6 +6,12 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie +# The block reward of coinbaseoutput.nValue (50) BTC/block matures after +# COINBASE_MATURITY (100) blocks. Therefore, after mining 101 blocks we expect +# node 0 to have a balance of (BLOCKS - COINBASE_MATURITY) * 50 BTC/block. +BLOCKS = 101 +BALANCE = (BLOCKS - 100) * 50 + class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): @@ -17,6 +23,7 @@ class TestBitcoinCli(BitcoinTestFramework): def run_test(self): """Main test logic""" + self.nodes[0].generate(BLOCKS) cli_response = self.nodes[0].cli("-version").send_cli() assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response @@ -35,7 +42,7 @@ class TestBitcoinCli(BitcoinTestFramework): user, password = get_auth_cookie(self.nodes[0].datadir, self.chain) self.log.info("Test -stdinrpcpass option") - assert_equal(0, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) + assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input="foo").echo) self.log.info("Test -stdin and -stdinrpcpass") @@ -51,10 +58,8 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Make sure that -getinfo with arguments fails") assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) - self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") + self.log.info("Test that -getinfo returns the expected network and blockchain info") cli_get_info = self.nodes[0].cli('-getinfo').send_cli() - if self.is_wallet_compiled(): - wallet_info = self.nodes[0].getwalletinfo() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() @@ -66,11 +71,15 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): - assert_equal(cli_get_info['balance'], wallet_info['balance']) + self.log.info("Test that -getinfo returns the expected wallet info") + assert_equal(cli_get_info['balance'], BALANCE) + wallet_info = self.nodes[0].getwalletinfo() assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) # unlocked_until is not tested because the wallet is not encrypted + else: + self.log.info("*** Wallet not compiled; -getinfo wallet tests skipped") if __name__ == '__main__': diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 188b130a57..b73d5784aa 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -14,10 +14,7 @@ from test_framework.messages import ( msg_filteradd, msg_filterclear, ) -from test_framework.mininode import ( - P2PInterface, - mininode_lock, -) +from test_framework.mininode import P2PInterface from test_framework.test_framework import BitcoinTestFramework @@ -69,18 +66,15 @@ class FilterTest(BitcoinTestFramework): filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] self.log.info('Check that we receive merkleblock and tx if the filter matches a tx in a block') - filter_node.merkleblock_received = False block_hash = self.nodes[0].generatetoaddress(1, filter_address)[0] txid = self.nodes[0].getblock(block_hash)['tx'][0] + filter_node.wait_for_merkleblock(block_hash) filter_node.wait_for_tx(txid) - assert filter_node.merkleblock_received self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block') - with mininode_lock: - filter_node.last_message.pop("merkleblock", None) filter_node.tx_received = False - self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress()) - filter_node.wait_for_merkleblock() + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_merkleblock(block_hash) assert not filter_node.tx_received self.log.info('Check that we not receive a tx if the filter does not match a mempool tx') diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 33b7060060..58be703bf6 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -34,7 +34,7 @@ class InvalidLocatorTest(BitcoinTestFramework): msg.locator.vHave = [int(node.getblockhash(i - 1), 16) for i in range(block_count, block_count - (MAX_LOCATOR_SZ), -1)] node.p2p.send_message(msg) if type(msg) == msg_getheaders: - node.p2p.wait_for_header(int(node.getbestblockhash(), 16)) + node.p2p.wait_for_header(node.getbestblockhash()) else: node.p2p.wait_for_block(int(node.getbestblockhash(), 16)) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 08e1f0bcd4..c574c1ab9f 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. -A node should never send anything other than VERSION/VERACK/REJECT until it's +A node should never send anything other than VERSION/VERACK until it's received a VERACK. This test connects to a node and sends it a few messages, trying to entice it @@ -39,7 +39,6 @@ class CLazyNode(P2PInterface): def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) - def on_reject(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) def on_addr(self, message): self.bad_message(message) def on_getdata(self, message): self.bad_message(message) @@ -69,8 +68,6 @@ class CNodeNoVersionBan(CLazyNode): for i in range(banscore): self.send_message(msg_verack()) - def on_reject(self, message): pass - # Node that never sends a version. This one just sits idle and hopes to receive # any message (it shouldn't!) class CNodeNoVersionIdle(CLazyNode): @@ -83,7 +80,6 @@ class CNodeNoVerackIdle(CLazyNode): self.version_received = False super().__init__() - def on_reject(self, message): pass def on_verack(self, message): pass # When version is received, don't reply with a verack. Instead, see if the # node will give us a message that it shouldn't. This is not an exhaustive diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py new file mode 100755 index 0000000000..f23d9ec556 --- /dev/null +++ b/test/functional/rpc_generateblock.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +'''Test generateblock rpc. +''' + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +class GenerateBlockTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + + self.log.info('Generate an empty block to address') + address = node.getnewaddress() + hash = node.generateblock(address, [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a descriptor') + hash = node.generateblock('addr(' + address + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + # Generate 110 blocks to spend + node.generatetoaddress(110, address) + + # Generate some extra mempool transactions to verify they don't get mined + for i in range(10): + node.sendtoaddress(address, 0.001) + + self.log.info('Generate block with txid') + txid = node.sendtoaddress(address, 1) + hash = node.generateblock(address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + utxos = node.listunspent(addresses=[address]) + raw = node.createrawtransaction([{'txid':utxos[0]['txid'], 'vout':utxos[0]['vout']}],[{address:1}]) + signed_raw = node.signrawtransactionwithwallet(raw)['hex'] + hash = node.generateblock(address, [signed_raw])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.gettransaction(txid)['hex'], signed_raw) + + self.log.info('Fail to generate block with out of order txs') + raw1 = node.createrawtransaction([{'txid':txid, 'vout':0}],[{address:0.9999}]) + signed_raw1 = node.signrawtransactionwithwallet(raw1)['hex'] + txid1 = node.sendrawtransaction(signed_raw1) + raw2 = node.createrawtransaction([{'txid':txid1, 'vout':0}],[{address:0.999}]) + signed_raw2 = node.signrawtransactionwithwallet(raw2)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', node.generateblock, address, [signed_raw2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', node.generateblock, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, node.generateblock, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', node.generateblock, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', node.generateblock, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', node.generateblock, child_descriptor, []) + +if __name__ == '__main__': + GenerateBlockTest().main() diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5f8fcc6fd8..45b49bcf9e 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1321,10 +1321,23 @@ class msg_headers: class msg_merkleblock: + __slots__ = ("merkleblock",) command = b"merkleblock" + def __init__(self, merkleblock=None): + if merkleblock is None: + self.merkleblock = CMerkleBlock() + else: + self.merkleblock = merkleblock + def deserialize(self, f): - pass # Placeholder for now + self.merkleblock.deserialize(f) + + def serialize(self): + return self.merkleblock.serialize() + + def __repr__(self): + return "msg_merkleblock(merkleblock=%s)" % (repr(self.merkleblock)) class msg_filterload: diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ad330f2a93..9f51bef946 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -339,7 +339,6 @@ class P2PInterface(P2PConnection): def on_merkleblock(self, message): pass def on_notfound(self, message): pass def on_pong(self, message): pass - def on_reject(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass @@ -393,18 +392,17 @@ class P2PInterface(P2PConnection): last_headers = self.last_message.get('headers') if not last_headers: return False - return last_headers.headers[0].rehash() == blockhash + return last_headers.headers[0].rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_merkleblock(self, timeout=60): + def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): assert self.is_connected last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: return False - # TODO change this method to take a hash value and only return true if the correct block has been received - return True + return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index faa2dee4ed..ee71de3310 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -173,6 +173,7 @@ BASE_SCRIPTS = [ 'wallet_importprunedfunds.py', 'p2p_leak_tx.py', 'rpc_signmessage.py', + 'rpc_generateblock.py', 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', |