diff options
246 files changed, 3785 insertions, 2106 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index fcf1200d35..836e21e6f0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -109,9 +109,9 @@ task: VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.3.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.3' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5' QTBASEDIR: 'C:\Qt_static' x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 40639dfe61..602b57fe43 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -18,7 +18,7 @@ m4_define([_CHECK_ATOMIC_testbody], [[ int main() { std::atomic<bool> lock{true}; - std::atomic_exchange(&lock, false); + lock.exchange(false); std::atomic<std::chrono::seconds> t{0s}; t.store(2s); @@ -34,6 +34,8 @@ m4_define([_CHECK_ATOMIC_testbody], [[ AC_DEFUN([CHECK_ATOMIC], [ AC_LANG_PUSH(C++) + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([whether std::atomic can be used without link library]) @@ -51,5 +53,6 @@ AC_DEFUN([CHECK_ATOMIC], [ ]) ]) + CXXFLAGS="$TEMP_CXXFLAGS" AC_LANG_POP ]) diff --git a/build_msvc/README.md b/build_msvc/README.md index 05cc2aad83..b9bebd369c 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -30,7 +30,7 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.3.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands: ```cmd diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 9477fb2d9f..97c530e19e 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -16,5 +16,5 @@ export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" # Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 7b714dff5c..d8c08fca39 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -14,4 +14,4 @@ export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" # Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5 -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CXXFLAGS='-fdebug-default-version=4'" # TODO enable GUI +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" # TODO enable GUI diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e1032ba6bd..1220b15d5a 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -35,18 +35,24 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then fi if [ "${RUN_TIDY}" = "true" ]; then + set -eo pipefail export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - CI_EXEC run-clang-tidy "${MAKEJOBS}" + ( CI_EXEC run-clang-tidy -quiet "${MAKEJOBS}" ) | grep -C5 "error" export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ " src/compat"\ + " src/dbwrapper.cpp"\ " src/init"\ + " src/kernel"\ + " src/node/chainstate.cpp"\ " src/policy/feerate.cpp"\ " src/policy/packages.cpp"\ " src/policy/settings.cpp"\ + " src/primitives/transaction.cpp"\ " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ " src/test/fuzz/txorphan.cpp"\ + " src/threadinterrupt.cpp"\ " src/util/bip32.cpp"\ " src/util/bytevectorhash.cpp"\ " src/util/error.cpp"\ diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index fcd56f533e..eb31edbce8 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do # shellcheck disable=SC2044 for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 525db9eded..1662f8f6a3 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}.exe; do # shellcheck disable=SC2044 for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do if (file "$b" | grep "Windows"); then diff --git a/configure.ac b/configure.ac index 0e1968f5c4..90e7f18515 100644 --- a/configure.ac +++ b/configure.ac @@ -87,9 +87,6 @@ else AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) fi -dnl Check if -latomic is required for <std::atomic> -CHECK_ATOMIC - dnl check if additional link flags are required for std::filesystem CHECK_FILESYSTEM @@ -887,6 +884,9 @@ AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD +dnl Check if -latomic is required for <std::atomic> +CHECK_ATOMIC + dnl The following macro will add the necessary defines to bitcoin-config.h, but dnl they also need to be passed down to any subprojects. Pull the results out of dnl the cache and add them to CPPFLAGS. @@ -1978,6 +1978,9 @@ AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) +AC_CONFIG_LINKS([src/qt/Makefile:src/qt/Makefile]) +AC_CONFIG_LINKS([src/qt/test/Makefile:src/qt/test/Makefile]) +AC_CONFIG_LINKS([src/test/Makefile:src/test/Makefile]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -2049,5 +2052,6 @@ echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPP echo " CXX = $CXX" echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp index ce7786f58c..919ffab102 100644 --- a/contrib/devtools/iwyu/bitcoin.core.imp +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -3,4 +3,5 @@ { include: [ "<bits/termios-c_lflag.h>", private, "<termios.h>", public ] }, { include: [ "<bits/termios-struct.h>", private, "<termios.h>", public ] }, { include: [ "<bits/termios-tcflow.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/chrono.h>", private, "<chrono>", public ] }, ] diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 12b563eac2..f93d6e26e8 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -132,12 +132,19 @@ chain for " target " development.")) (define base-gcc gcc-10) (define base-linux-kernel-headers linux-libre-headers-5.15) +;; https://gcc.gnu.org/install/configure.html +(define (hardened-gcc gcc) + (package-with-extra-configure-variable ( + package-with-extra-configure-variable gcc + "--enable-default-ssp" "yes") + "--enable-default-pie" "yes")) + (define* (make-bitcoin-cross-toolchain target #:key (base-gcc-for-libc base-gcc) (base-kernel-headers base-linux-kernel-headers) - (base-libc (make-glibc-without-werror glibc-2.24)) - (base-gcc (make-gcc-rpath-link base-gcc))) + (base-libc (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.24))) + (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -520,6 +527,12 @@ inspecting signatures in Mach-O binaries.") (define (make-glibc-without-werror glibc) (package-with-extra-configure-variable glibc "enable_werror" "no")) +(define (make-glibc-with-stack-protector glibc) + (package-with-extra-configure-variable glibc "--enable-stack-protector" "all")) + +(define (make-glibc-with-bind-now glibc) + (package-with-extra-configure-variable glibc "--enable-bind-now" "yes")) + (define-public glibc-2.24 (package (inherit glibc-2.31) @@ -607,8 +620,8 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "-linux-") (list (cond ((string-contains target "riscv64-") (make-bitcoin-cross-toolchain target - #:base-libc (make-glibc-without-werror glibc-2.27/bitcoin-patched) - #:base-kernel-headers base-linux-kernel-headers)) + #:base-libc (make-glibc-with-stack-protector + (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.27/bitcoin-patched))))) (else (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") diff --git a/contrib/signet/miner b/contrib/signet/miner index 61415cb2dd..fdcd20ae3b 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import base64 import json import logging import math @@ -15,14 +14,13 @@ import sys import time import subprocess -from io import BytesIO - PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, ser_compact_size, ser_string, ser_uint256, tx_from_hex # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_binary, from_hex, ser_string, ser_uint256, tx_from_hex # noqa: E402 +from test_framework.psbt import PSBT, PSBTMap, PSBT_GLOBAL_UNSIGNED_TX, PSBT_IN_FINAL_SCRIPTSIG, PSBT_IN_FINAL_SCRIPTWITNESS, PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_SIGHASH_TYPE # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -34,89 +32,6 @@ SIGNET_HEADER = b"\xec\xc7\xda\xa2" PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") -# #### some helpers that could go into test_framework - -# like from_hex, but without the hex part -def FromBinary(cls, stream): - """deserialize a binary stream (or bytes object) into an object""" - # handle bytes object by turning it into a stream - was_bytes = isinstance(stream, bytes) - if was_bytes: - stream = BytesIO(stream) - obj = cls() - obj.deserialize(stream) - if was_bytes: - assert len(stream.read()) == 0 - return obj - -class PSBTMap: - """Class for serializing and deserializing PSBT maps""" - - def __init__(self, map=None): - self.map = map if map is not None else {} - - def deserialize(self, f): - m = {} - while True: - k = deser_string(f) - if len(k) == 0: - break - v = deser_string(f) - if len(k) == 1: - k = k[0] - assert k not in m - m[k] = v - self.map = m - - def serialize(self): - m = b"" - for k,v in self.map.items(): - if isinstance(k, int) and 0 <= k and k <= 255: - k = bytes([k]) - m += ser_compact_size(len(k)) + k - m += ser_compact_size(len(v)) + v - m += b"\x00" - return m - -class PSBT: - """Class for serializing and deserializing PSBTs""" - - def __init__(self): - self.g = PSBTMap() - self.i = [] - self.o = [] - self.tx = None - - def deserialize(self, f): - assert f.read(5) == b"psbt\xff" - self.g = FromBinary(PSBTMap, f) - assert 0 in self.g.map - self.tx = FromBinary(CTransaction, self.g.map[0]) - self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] - self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] - return self - - def serialize(self): - assert isinstance(self.g, PSBTMap) - assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) - assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) - assert 0 in self.g.map - tx = FromBinary(CTransaction, self.g.map[0]) - assert len(tx.vin) == len(self.i) - assert len(tx.vout) == len(self.o) - - psbt = [x.serialize() for x in [self.g] + self.i + self.o] - return b"psbt\xff" + b"".join(psbt) - - def to_base64(self): - return base64.b64encode(self.serialize()).decode("utf8") - - @classmethod - def from_base64(cls, b64psbt): - return FromBinary(cls, base64.b64decode(b64psbt)) - -# ##### - def create_coinbase(height, value, spk): cb = CTransaction() cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] @@ -159,11 +74,11 @@ def signet_txs(block, challenge): def do_createpsbt(block, signme, spendme): psbt = PSBT() - psbt.g = PSBTMap( {0: signme.serialize(), + psbt.g = PSBTMap( {PSBT_GLOBAL_UNSIGNED_TX: signme.serialize(), PSBT_SIGNET_BLOCK: block.serialize() } ) - psbt.i = [ PSBTMap( {0: spendme.serialize(), - 3: bytes([1,0,0,0])}) + psbt.i = [ PSBTMap( {PSBT_IN_NON_WITNESS_UTXO: spendme.serialize(), + PSBT_IN_SIGHASH_TYPE: bytes([1,0,0,0])}) ] psbt.o = [ PSBTMap() ] return psbt.to_base64() @@ -175,10 +90,10 @@ def do_decode_psbt(b64psbt): assert len(psbt.tx.vout) == 1 assert PSBT_SIGNET_BLOCK in psbt.g.map - scriptSig = psbt.i[0].map.get(7, b"") - scriptWitness = psbt.i[0].map.get(8, b"\x00") + scriptSig = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"") + scriptWitness = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00") - return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness def finish_block(block, signet_solution, grind_cmd): block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 6efe49254b..d6856b4274 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -65,12 +65,6 @@ obj:*/libdb_cxx-*.so } { - Suppress leaks on init - Memcheck:Leak - ... - fun:_Z11AppInitMainR11NodeContext -} -{ Suppress leaks on shutdown Memcheck:Leak ... @@ -83,21 +77,6 @@ obj:/usr/lib64/libgdk-3.so.0.2404.7 } { - Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - fun:_ZN7leveldbL10InitModuleEv -} -{ - Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_ZN7leveldbL14InitDefaultEnvEv -} -{ Suppress leveldb leak Memcheck:Leak match-leak-kinds: reachable diff --git a/depends/config.site.in b/depends/config.site.in index 189330c42d..f7e770343c 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -105,7 +105,7 @@ PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PA if test -n "@AR@"; then AR="@AR@" - ac_cv_path_ac_pt_AR="${AR}" + ac_cv_path_AR="${AR}" fi if test -n "@RANLIB@"; then @@ -126,17 +126,17 @@ fi if test "@host_os@" = darwin; then if test -n "@OTOOL@"; then OTOOL="@OTOOL@" - ac_cv_path_ac_pt_OTOOL="${OTOOL}" + ac_cv_path_OTOOL="${OTOOL}" fi if test -n "@INSTALL_NAME_TOOL@"; then INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" - ac_cv_path_ac_pt_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" + ac_cv_path_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" fi if test -n "@DSYMUTIL@"; then DSYMUTIL="@DSYMUTIL@" - ac_cv_path_ac_pt_DSYMUTIL="${DSYMUTIL}" + ac_cv_path_DSYMUTIL="${DSYMUTIL}" fi fi diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index b101043439..635d3d16da 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -5,6 +5,10 @@ ifneq ($(LTO),) linux_CFLAGS += -flto linux_CXXFLAGS += -flto linux_LDFLAGS += -flto + +linux_AR = $(host_toolchain)gcc-ar +linux_NM = $(host_toolchain)gcc-nm +linux_RANLIB = $(host_toolchain)gcc-ranlib endif linux_release_CFLAGS=-O2 diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index b98f9ab7ac..fc1cc1afbe 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -9,6 +9,10 @@ ifneq ($(LTO),) mingw32_CFLAGS += -flto mingw32_CXXFLAGS += -flto mingw32_LDFLAGS += -flto + +mingw32_AR = $(host_toolchain)gcc-ar +mingw32_NM = $(host_toolchain)gcc-nm +mingw32_RANLIB = $(host_toolchain)gcc-ranlib endif mingw32_release_CFLAGS=-O2 diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk index 8342dcc6ed..14121dca20 100644 --- a/depends/hosts/netbsd.mk +++ b/depends/hosts/netbsd.mk @@ -5,6 +5,10 @@ ifneq ($(LTO),) netbsd_CFLAGS += -flto netbsd_CXXFLAGS += -flto netbsd_LDFLAGS += -flto + +netbsd_AR = $(host_toolchain)gcc-ar +netbsd_NM = $(host_toolchain)gcc-nm +netbsd_RANLIB = $(host_toolchain)gcc-ranlib endif netbsd_CXXFLAGS=$(netbsd_CFLAGS) diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 50791ebc6e..bb203d06f8 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,14 +1,18 @@ package=expat -$(package)_version=2.4.1 +$(package)_version=2.4.8 $(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/ $(package)_file_name=$(package)-$($(package)_version).tar.xz -$(package)_sha256_hash=cf032d0dba9b928636548e32b327a2d66b1aab63c4f4a13dd132c2d1d2f2fb6a +$(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df479dcaf25 +# -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional +# definitions in endian.h, which are required for a working +# endianess check in configure when building with -flto. define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts += --without-xmlwf $(package)_config_opts_linux=--with-pic + $(package)_cppflags += -D_DEFAULT_SOURCE endef define $(package)_config_cmds diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index fa30e80f5c..036eaf6560 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=$(package)-$($(package)_version).tar.xz $(package)_sha256_hash=a55ed6db98d43469801262d81dc2572ed124edc3db31059d4e9916eb9f844c34 $(package)_dependencies=xcb_proto libXau +$(package)_patches = remove_pthread_stubs.patch define $(package)_set_vars $(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd @@ -20,7 +21,7 @@ endef define $(package)_preprocess_cmds cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux && \ - sed "s/pthread-stubs//" -i configure + patch -p1 -i $($(package)_patch_dir)/remove_pthread_stubs.patch endef define $(package)_config_cmds diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 44108925a4..51a95f48ef 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -1,6 +1,6 @@ package=native_ds_store $(package)_version=1.3.0 -$(package)_download_path=https://github.com/al45tair/ds_store/archive/ +$(package)_download_path=https://github.com/dmgbuild/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 783f87ca7c..ddd631186e 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,6 +1,6 @@ package=native_mac_alias $(package)_version=2.2.0 -$(package)_download_path=https://github.com/al45tair/mac_alias/archive/ +$(package)_download_path=https://github.com/dmgbuild/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 7e268fe9a5..148a18cf53 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.3 +$(package)_version=5.15.5 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=26394ec9375d52c1592bd7b689b1619c6b8dbe9b6f91fdd5c355589787f3a0b6 +$(package)_sha256_hash=0c42c799aa7c89e479a07c451bf5a301e291266ba789e81afc18f95049524edc $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -17,16 +17,17 @@ $(package)_patches += fix_montery_include.patch $(package)_patches += fix_android_jni_static.patch $(package)_patches += dont_hardcode_pwd.patch $(package)_patches += qtbase-moc-ignore-gcc-macro.patch -$(package)_patches += fix_limits_header.patch $(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch $(package)_patches += duplicate_lcqpafonts.patch +$(package)_patches += fast_fixed_dtoa_no_optimize.patch +$(package)_patches += guix_cross_lib_path.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=5d7869f670a135ad0986e266813b9dd5bbae2b09577338f9cdf8904d4af52db0 +$(package)_qttranslations_sha256_hash=c92af4171397a0ed272330b4fa0669790fcac8d050b07c8b8cc565ebeba6735e $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=463b2fe71a085e7ab4e39333ae360ab0ec857b966d7a08f752c427e5df55f90d +$(package)_qttools_sha256_hash=6d0778b71b2742cb527561791d1d3d255366163d54a10f78c683a398f09ffc6c $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -152,6 +153,9 @@ $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl $(package)_config_opts_linux += -no-feature-vulkan $(package)_config_opts_linux += -dbus-runtime +ifneq ($(LTO),) +$(package)_config_opts_linux += -ltcg +endif $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_i686_linux = -xplatform linux-g++-32 ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) @@ -173,6 +177,7 @@ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(pack $(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_mingw32 += -pch @@ -243,11 +248,12 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ + patch -p1 -i $($(package)_patch_dir)/fast_fixed_dtoa_no_optimize.patch && \ + patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ @@ -257,8 +263,7 @@ define $(package)_preprocess_cmds echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ sed -i.old "s|QMAKE_CC = \$$$$\$$$${CROSS_COMPILE}clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf + sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf endef define $(package)_config_cmds diff --git a/depends/patches/libxcb/remove_pthread_stubs.patch b/depends/patches/libxcb/remove_pthread_stubs.patch new file mode 100644 index 0000000000..1f32dea527 --- /dev/null +++ b/depends/patches/libxcb/remove_pthread_stubs.patch @@ -0,0 +1,12 @@ +Remove uneeded pthread-stubs dependency +--- a/configure ++++ b/configure +@@ -19695,7 +19695,7 @@ fi + NEEDED="xau >= 0.99.2" + case $host_os in + linux*) ;; +- *) NEEDED="$NEEDED pthread-stubs" ;; ++ *) NEEDED="$NEEDED" ;; + esac + + pkg_failed=no diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch index 5c1e030fa4..a66426877a 100644 --- a/depends/patches/qt/dont_hardcode_x86_64.patch +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -73,7 +73,7 @@ diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/defaul index 92a9112bca6..d888731ec8d 100644 --- old/qtbase/mkspecs/features/mac/default_post.prf +++ new/qtbase/mkspecs/features/mac/default_post.prf -@@ -90,6 +90,11 @@ app_extension_api_only { +@@ -95,6 +95,11 @@ app_extension_api_only { QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION } @@ -85,7 +85,7 @@ index 92a9112bca6..d888731ec8d 100644 macx-xcode { qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ -@@ -145,9 +150,6 @@ macx-xcode { +@@ -150,9 +155,6 @@ macx-xcode { simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch new file mode 100644 index 0000000000..d4d6539f56 --- /dev/null +++ b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch @@ -0,0 +1,20 @@ +Modify the optimisation flags for FastFixedDtoa. +This fixes a non-determinism issue in the asm produced for +this function when cross-compiling on x86_64 and aarch64 for +the arm-linux-gnueabihf HOST. + +--- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h ++++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h +@@ -48,9 +48,12 @@ namespace double_conversion { + // + // This method only works for some parameters. If it can't handle the input it + // returns false. The output is null-terminated when the function succeeds. ++#pragma GCC push_options ++#pragma GCC optimize ("-O1") + bool FastFixedDtoa(double v, int fractional_count, + Vector<char> buffer, int* length, int* decimal_point); + ++#pragma GCC pop_options + } // namespace double_conversion + + #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 22a4d5ab0e..936b82e152 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -934,6 +934,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) +@@ -943,6 +943,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch deleted file mode 100644 index 258128c0ca..0000000000 --- a/depends/patches/qt/fix_limits_header.patch +++ /dev/null @@ -1,33 +0,0 @@ -Fix compiling with GCC 11 - -Upstream: - - bug report: https://bugreports.qt.io/browse/QTBUG-89977 - - fix in Qt 6.1: 813a928c7c3cf98670b6043149880ed5c955efb9 - ---- old/qtbase/src/corelib/text/qbytearraymatcher.h -+++ new/qtbase/src/corelib/text/qbytearraymatcher.h -@@ -42,6 +42,8 @@ - - #include <QtCore/qbytearray.h> - -+#include <limits> -+ - QT_BEGIN_NAMESPACE - - - -Upstream fix and backports: - - Qt 6.1: 3eab20ad382569cb2c9e6ccec2322c3d08c0f716 - - Qt 6.2: 380294a5971da85010a708dc23b0edec192cbf27 - - Qt 6.3: 2b2b3155d9f6ba1e4f859741468fbc47db09292b - ---- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h -+++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h -@@ -55,6 +55,7 @@ - - #include <tuple> - #include <array> -+#include <limits> - - QT_BEGIN_NAMESPACE - diff --git a/depends/patches/qt/guix_cross_lib_path.patch b/depends/patches/qt/guix_cross_lib_path.patch new file mode 100644 index 0000000000..0c67743dc1 --- /dev/null +++ b/depends/patches/qt/guix_cross_lib_path.patch @@ -0,0 +1,17 @@ +Facilitate guix building with CROSS_LIBRARY_PATH + +See discussion in https://github.com/bitcoin/bitcoin/pull/15277. + +--- a/qtbase/mkspecs/features/toolchain.prf ++++ b/qtbase/mkspecs/features/toolchain.prf +@@ -231,8 +231,8 @@ isEmpty($${target_prefix}.INCDIRS) { + add_libraries = false + for (line, output) { + line ~= s/^[ \\t]*// # remove leading spaces +- contains(line, "LIBRARY_PATH=.*") { +- line ~= s/^LIBRARY_PATH=// # remove leading LIBRARY_PATH= ++ contains(line, "(CROSS_)?LIBRARY_PATH=.*") { ++ line ~= s/^(CROSS_)?LIBRARY_PATH=// # remove leading (CROSS_)?LIBRARY_PATH= + equals(QMAKE_HOST.os, Windows): \ + paths = $$split(line, ;) + else: \ diff --git a/doc/dependencies.md b/doc/dependencies.md index 697d432520..8b8259ab0a 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [3.4.4](https://github.com/bitcoin/bitcoin/pull/6373) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.3](https://github.com/bitcoin/bitcoin/pull/24668) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.5](https://github.com/bitcoin/bitcoin/pull/25719) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/release-notes-24148.md b/doc/release-notes-24148.md new file mode 100644 index 0000000000..f7a0fd6fa1 --- /dev/null +++ b/doc/release-notes-24148.md @@ -0,0 +1,23 @@ +Notable changes +=============== + +Wallet +------ + +- The `wsh()` output descriptor was extended with Miniscript support. You can import Miniscript + descriptors for P2WSH in a watchonly wallet to track coins, but you can't spend from them using + the Bitcoin Core wallet yet. + You can find more about Miniscript on the [reference website](https://bitcoin.sipa.be/miniscript/). + + +Low-level changes +================= + +RPC +--- + +- The `deriveaddresses`, `getdescriptorinfo`, `importdescriptors` and `scantxoutset` commands now + accept Miniscript expression within a `wsh()` descriptor. + +- The `getaddressinfo`, `decodescript`, `listdescriptors` and `listunspent` commands may now output + a Miniscript descriptor inside a `wsh()` where a `wsh(raw())` descriptor was previously returned. diff --git a/src/.clang-tidy b/src/.clang-tidy index e9807d4cb7..b9371b147b 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,13 +1,17 @@ Checks: ' -*, bugprone-argument-comment, +misc-unused-using-decls, modernize-use-default-member-init, modernize-use-nullptr, readability-redundant-declaration, +readability-redundant-string-init, ' WarningsAsErrors: ' bugprone-argument-comment, +misc-unused-using-decls, modernize-use-default-member-init, modernize-use-nullptr, readability-redundant-declaration, +readability-redundant-string-init, ' diff --git a/src/Makefile.am b/src/Makefile.am index d293775fc3..23bc180095 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,9 +133,9 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ common/bloom.h \ - compat.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/compat.h \ compat/cpuid.h \ compat/endian.h \ compressor.h \ @@ -171,12 +171,14 @@ BITCOIN_CORE_H = \ interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ + kernel/chain.h \ kernel/chainstatemanager_opts.h \ kernel/checks.h \ kernel/coinstats.h \ kernel/context.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ + kernel/mempool_persist.h \ key.h \ key_io.h \ logging.h \ @@ -198,6 +200,7 @@ BITCOIN_CORE_H = \ node/chainstate.h \ node/coin.h \ node/context.h \ + node/mempool_persist_args.h \ node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ @@ -363,9 +366,11 @@ libbitcoin_node_a_SOURCES = \ index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ + kernel/chain.cpp \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ mempool_args.cpp \ net.cpp \ @@ -379,6 +384,7 @@ libbitcoin_node_a_SOURCES = \ node/context.cpp \ node/eviction.cpp \ node/interfaces.cpp \ + node/mempool_persist_args.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ @@ -878,9 +884,11 @@ libbitcoinkernel_la_SOURCES = \ flatfile.cpp \ fs.cpp \ hash.cpp \ + kernel/chain.cpp \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b806b62d5b..f52964b033 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -117,6 +117,7 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/rbf_tests.cpp \ test/rest_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ @@ -167,6 +168,7 @@ BITCOIN_TESTS += \ wallet/test/wallet_crypto_tests.cpp \ wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ + wallet/test/availablecoins_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp @@ -373,7 +375,7 @@ endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if ENABLE_TESTS -UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester noinst_PROGRAMS += $(UNIVALUE_TESTS) TESTS += $(UNIVALUE_TESTS) @@ -382,11 +384,6 @@ univalue_test_unitester_LDADD = $(LIBUNIVALUE) univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) -univalue_test_no_nul_LDADD = $(LIBUNIVALUE) -univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) - univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) univalue_test_object_LDADD = $(LIBUNIVALUE) univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 8922dda3ad..b43816636f 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ + test/fuzz/mempool_utils.h \ test/fuzz/util.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) diff --git a/src/addrman.cpp b/src/addrman.cpp index 204bb544c5..857ad73b2f 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -29,19 +29,19 @@ static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; /** Maximum number of times an address can occur in the new table */ static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; /** How old addresses can maximally be */ -static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; +static constexpr auto ADDRMAN_HORIZON{30 * 24h}; /** After how many failed attempts we give up on a new node */ static constexpr int32_t ADDRMAN_RETRIES{3}; /** How many successive failures are allowed ... */ static constexpr int32_t ADDRMAN_MAX_FAILURES{10}; -/** ... in at least this many days */ -static constexpr int64_t ADDRMAN_MIN_FAIL_DAYS{7}; +/** ... in at least this duration */ +static constexpr auto ADDRMAN_MIN_FAIL{7 * 24h}; /** How recent a successful connection should be before we allow an address to be evicted from tried */ -static constexpr int64_t ADDRMAN_REPLACEMENT_HOURS{4}; +static constexpr auto ADDRMAN_REPLACEMENT{4h}; /** The maximum number of tried addr collisions to store */ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; -/** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ -static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes +/** The maximum time we'll spend trying to resolve a tried table collision */ +static constexpr auto ADDRMAN_TEST_WINDOW{40min}; int AddrInfo::GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { @@ -64,36 +64,39 @@ int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) con return hash1 % ADDRMAN_BUCKET_SIZE; } -bool AddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(NodeSeconds now) const { - if (nNow - nLastTry <= 60) { // never remove things tried in the last minute + if (now - m_last_try <= 1min) { // never remove things tried in the last minute return false; } - if (nTime > nNow + 10 * 60) // came in a flying DeLorean + if (nTime > now + 10min) { // came in a flying DeLorean return true; + } - if (nNow - nTime > ADDRMAN_HORIZON_DAYS * 24 * 60 * 60) { // not seen in recent history + if (now - nTime > ADDRMAN_HORIZON) { // not seen in recent history return true; } - if (nLastSuccess == 0 && nAttempts >= ADDRMAN_RETRIES) // tried N times and never a success + if (TicksSinceEpoch<std::chrono::seconds>(m_last_success) == 0 && nAttempts >= ADDRMAN_RETRIES) { // tried N times and never a success return true; + } - if (nNow - nLastSuccess > ADDRMAN_MIN_FAIL_DAYS * 24 * 60 * 60 && nAttempts >= ADDRMAN_MAX_FAILURES) // N successive failures in the last week + if (now - m_last_success > ADDRMAN_MIN_FAIL && nAttempts >= ADDRMAN_MAX_FAILURES) { // N successive failures in the last week return true; + } return false; } -double AddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(NodeSeconds now) const { double fChance = 1.0; - int64_t nSinceLastTry = std::max<int64_t>(nNow - nLastTry, 0); // deprioritize very recent attempts away - if (nSinceLastTry < 60 * 10) + if (now - m_last_try < 10min) { fChance *= 0.01; + } // deprioritize 66% after each failed attempt, but at most 1/28th to avoid the search taking forever or overly penalizing outages. fChance *= pow(0.66, std::min(nAttempts, 8)); @@ -540,7 +543,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) info.fInTried = true; } -bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) { AssertLockHeld(cs); @@ -552,15 +555,15 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ // Do not set a penalty for a source's self-announcement if (addr == source) { - nTimePenalty = 0; + time_penalty = 0s; } if (pinfo) { // periodically update nTime - bool fCurrentlyOnline = (GetAdjustedTime() - addr.nTime < 24 * 60 * 60); - int64_t nUpdateInterval = (fCurrentlyOnline ? 60 * 60 : 24 * 60 * 60); - if (pinfo->nTime < addr.nTime - nUpdateInterval - nTimePenalty) { - pinfo->nTime = std::max((int64_t)0, addr.nTime - nTimePenalty); + const bool currently_online{AdjustedTime() - addr.nTime < 24h}; + const auto update_interval{currently_online ? 1h : 24h}; + if (pinfo->nTime < addr.nTime - update_interval - time_penalty) { + pinfo->nTime = std::max(NodeSeconds{0s}, addr.nTime - time_penalty); } // add services @@ -587,7 +590,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ return false; } else { pinfo = Create(addr, source, &nId); - pinfo->nTime = std::max((int64_t)0, (int64_t)pinfo->nTime - nTimePenalty); + pinfo->nTime = std::max(NodeSeconds{0s}, pinfo->nTime - time_penalty); nNew++; } @@ -617,13 +620,13 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ return fInsert; } -bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) +bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSeconds time) { AssertLockHeld(cs); int nId; - nLastGood = nTime; + m_last_good = time; AddrInfo* pinfo = Find(addr, &nId); @@ -633,8 +636,8 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT AddrInfo& info = *pinfo; // update info - info.nLastSuccess = nTime; - info.nLastTry = nTime; + info.m_last_success = time; + info.m_last_try = time; info.nAttempts = 0; // nTime is not updated here, to avoid leaking information about // currently-connected peers. @@ -671,11 +674,11 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT } } -bool AddrManImpl::Add_(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add_(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { int added{0}; for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) { - added += AddSingle(*it, source, nTimePenalty) ? 1 : 0; + added += AddSingle(*it, source, time_penalty) ? 1 : 0; } if (added > 0) { LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); @@ -683,7 +686,7 @@ bool AddrManImpl::Add_(const std::vector<CAddress> &vAddr, const CNetAddr& sourc return added > 0; } -void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) { AssertLockHeld(cs); @@ -696,14 +699,14 @@ void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTi AddrInfo& info = *pinfo; // update info - info.nLastTry = nTime; - if (fCountFailure && info.nLastCountAttempt < nLastGood) { - info.nLastCountAttempt = nTime; + info.m_last_try = time; + if (fCountFailure && info.m_last_count_attempt < m_last_good) { + info.m_last_count_attempt = time; info.nAttempts++; } } -std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const +std::pair<CAddress, NodeSeconds> AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); @@ -736,7 +739,7 @@ std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); - return {info, info.nLastTry}; + return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; @@ -764,7 +767,7 @@ std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); - return {info, info.nLastTry}; + return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; @@ -785,7 +788,7 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct } // gather a list of random nodes, skipping those of low quality - const int64_t now{GetAdjustedTime()}; + const auto now{AdjustedTime()}; std::vector<CAddress> addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { if (addresses.size() >= nNodes) @@ -810,7 +813,7 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct return addresses; } -void AddrManImpl::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, NodeSeconds time) { AssertLockHeld(cs); @@ -823,9 +826,10 @@ void AddrManImpl::Connected_(const CService& addr, int64_t nTime) AddrInfo& info = *pinfo; // update info - int64_t nUpdateInterval = 20 * 60; - if (nTime - info.nTime > nUpdateInterval) - info.nTime = nTime; + const auto update_interval{20min}; + if (time - info.nTime > update_interval) { + info.nTime = time; + } } void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) @@ -870,22 +874,22 @@ void AddrManImpl::ResolveCollisions_() int id_old = vvTried[tried_bucket][tried_bucket_pos]; AddrInfo& info_old = mapInfo[id_old]; - const auto current_time{GetAdjustedTime()}; + const auto current_time{AdjustedTime()}; // Has successfully connected in last X hours - if (current_time - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { + if (current_time - info_old.m_last_success < ADDRMAN_REPLACEMENT) { erase_collision = true; - } else if (current_time - info_old.nLastTry < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { // attempted to connect and failed in last X hours + } else if (current_time - info_old.m_last_try < ADDRMAN_REPLACEMENT) { // attempted to connect and failed in last X hours // Give address at least 60 seconds to successfully connect - if (current_time - info_old.nLastTry > 60) { + if (current_time - info_old.m_last_try > 60s) { LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); // Replaces an existing address already in the tried table with the new address Good_(info_new, false, current_time); erase_collision = true; } - } else if (current_time - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + } else if (current_time - info_new.m_last_success > ADDRMAN_TEST_WINDOW) { // If the collision hasn't resolved in some reasonable amount of time, // just evict the old entry -- we must not be able to // connect to it for some reason. @@ -894,7 +898,7 @@ void AddrManImpl::ResolveCollisions_() erase_collision = true; } } else { // Collision is not actually a collision anymore - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, AdjustedTime()); erase_collision = true; } } @@ -907,7 +911,7 @@ void AddrManImpl::ResolveCollisions_() } } -std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() +std::pair<CAddress, NodeSeconds> AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); @@ -932,7 +936,7 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; - return {info_old, info_old.nLastTry}; + return {info_old, info_old.m_last_try}; } std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& addr) @@ -990,8 +994,9 @@ int AddrManImpl::CheckAddrman() const int n = entry.first; const AddrInfo& info = entry.second; if (info.fInTried) { - if (!info.nLastSuccess) + if (!TicksSinceEpoch<std::chrono::seconds>(info.m_last_success)) { return -1; + } if (info.nRefCount) return -2; setTried.insert(n); @@ -1008,10 +1013,12 @@ int AddrManImpl::CheckAddrman() const } if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) return -14; - if (info.nLastTry < 0) + if (info.m_last_try < NodeSeconds{0s}) { return -6; - if (info.nLastSuccess < 0) + } + if (info.m_last_success < NodeSeconds{0s}) { return -8; + } } if (setTried.size() != (size_t)nTried) @@ -1067,29 +1074,29 @@ size_t AddrManImpl::size() const return vRandom.size(); } -bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { LOCK(cs); Check(); - auto ret = Add_(vAddr, source, nTimePenalty); + auto ret = Add_(vAddr, source, time_penalty); Check(); return ret; } -bool AddrManImpl::Good(const CService& addr, int64_t nTime) +bool AddrManImpl::Good(const CService& addr, NodeSeconds time) { LOCK(cs); Check(); - auto ret = Good_(addr, /*test_before_evict=*/true, nTime); + auto ret = Good_(addr, /*test_before_evict=*/true, time); Check(); return ret; } -void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) { LOCK(cs); Check(); - Attempt_(addr, fCountFailure, nTime); + Attempt_(addr, fCountFailure, time); Check(); } @@ -1101,7 +1108,7 @@ void AddrManImpl::ResolveCollisions() Check(); } -std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision() +std::pair<CAddress, NodeSeconds> AddrManImpl::SelectTriedCollision() { LOCK(cs); Check(); @@ -1110,7 +1117,7 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision() return ret; } -std::pair<CAddress, int64_t> AddrManImpl::Select(bool newOnly) const +std::pair<CAddress, NodeSeconds> AddrManImpl::Select(bool newOnly) const { LOCK(cs); Check(); @@ -1128,11 +1135,11 @@ std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, return addresses; } -void AddrManImpl::Connected(const CService& addr, int64_t nTime) +void AddrManImpl::Connected(const CService& addr, NodeSeconds time) { LOCK(cs); Check(); - Connected_(addr, nTime); + Connected_(addr, time); Check(); } @@ -1184,19 +1191,19 @@ size_t AddrMan::size() const return m_impl->size(); } -bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { - return m_impl->Add(vAddr, source, nTimePenalty); + return m_impl->Add(vAddr, source, time_penalty); } -bool AddrMan::Good(const CService& addr, int64_t nTime) +bool AddrMan::Good(const CService& addr, NodeSeconds time) { - return m_impl->Good(addr, nTime); + return m_impl->Good(addr, time); } -void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrMan::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) { - m_impl->Attempt(addr, fCountFailure, nTime); + m_impl->Attempt(addr, fCountFailure, time); } void AddrMan::ResolveCollisions() @@ -1204,12 +1211,12 @@ void AddrMan::ResolveCollisions() m_impl->ResolveCollisions(); } -std::pair<CAddress, int64_t> AddrMan::SelectTriedCollision() +std::pair<CAddress, NodeSeconds> AddrMan::SelectTriedCollision() { return m_impl->SelectTriedCollision(); } -std::pair<CAddress, int64_t> AddrMan::Select(bool newOnly) const +std::pair<CAddress, NodeSeconds> AddrMan::Select(bool newOnly) const { return m_impl->Select(newOnly); } @@ -1219,9 +1226,9 @@ std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std return m_impl->GetAddr(max_addresses, max_pct, network); } -void AddrMan::Connected(const CService& addr, int64_t nTime) +void AddrMan::Connected(const CService& addr, NodeSeconds time) { - m_impl->Connected(addr, nTime); + m_impl->Connected(addr, time); } void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) diff --git a/src/addrman.h b/src/addrman.h index a0063e8a9c..b70c6a48ad 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -11,6 +11,7 @@ #include <protocol.h> #include <streams.h> #include <timedata.h> +#include <util/time.h> #include <cstdint> #include <memory> @@ -107,23 +108,23 @@ public: * * @param[in] vAddr Address records to attempt to add. * @param[in] source The address of the node that sent us these addr records. - * @param[in] nTimePenalty A "time penalty" to apply to the address record's nTime. If a peer + * @param[in] time_penalty A "time penalty" to apply to the address record's nTime. If a peer * sends us an address record with nTime=n, then we'll add it to our - * addrman with nTime=(n - nTimePenalty). + * addrman with nTime=(n - time_penalty). * @return true if at least one address is successfully added. */ - bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty = 0s); /** * Mark an address record as accessible and attempt to move it to addrman's tried table. * * @param[in] addr Address record to attempt to move to tried table. - * @param[in] nTime The time that we were last connected to this peer. + * @param[in] time The time that we were last connected to this peer. * @return true if the address is successfully moved from the new table to the tried table. */ - bool Good(const CService& addr, int64_t nTime = GetAdjustedTime()); + bool Good(const CService& addr, NodeSeconds time = AdjustedTime()); //! Mark an entry as connection attempted to. - void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time = AdjustedTime()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. void ResolveCollisions(); @@ -133,18 +134,18 @@ public: * attempting to evict. * * @return CAddress The record for the selected tried peer. - * int64_t The last time we attempted to connect to that peer. + * seconds The last time we attempted to connect to that peer. */ - std::pair<CAddress, int64_t> SelectTriedCollision(); + std::pair<CAddress, NodeSeconds> SelectTriedCollision(); /** * Choose an address to connect to. * * @param[in] newOnly Whether to only select addresses from the new table. * @return CAddress The record for the selected peer. - * int64_t The last time we attempted to connect to that peer. + * seconds The last time we attempted to connect to that peer. */ - std::pair<CAddress, int64_t> Select(bool newOnly = false) const; + std::pair<CAddress, NodeSeconds> Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. @@ -166,9 +167,9 @@ public: * not leak information about currently connected peers. * * @param[in] addr The address of the peer we were connected to - * @param[in] nTime The time that we were last connected to this peer + * @param[in] time The time that we were last connected to this peer */ - void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); + void Connected(const CService& addr, NodeSeconds time = AdjustedTime()); //! Update an entry's service bits. void SetServices(const CService& addr, ServiceFlags nServices); diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 9d98cdde54..a73a026940 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -11,7 +11,9 @@ #include <protocol.h> #include <serialize.h> #include <sync.h> +#include <timedata.h> #include <uint256.h> +#include <util/time.h> #include <cstdint> #include <optional> @@ -38,16 +40,16 @@ class AddrInfo : public CAddress { public: //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; + NodeSeconds m_last_try{0s}; //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; + NodeSeconds m_last_count_attempt{0s}; //! where knowledge about this address first came from CNetAddr source; //! last successful connection by us - int64_t nLastSuccess{0}; + NodeSeconds m_last_success{0s}; //! connection attempts since last successful attempt int nAttempts{0}; @@ -64,7 +66,7 @@ public: SERIALIZE_METHODS(AddrInfo, obj) { READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + READWRITE(obj.source, Using<ChronoFormatter<int64_t>>(obj.m_last_success), obj.nAttempts); } AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) @@ -91,10 +93,10 @@ public: int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + bool IsTerrible(NodeSeconds now = AdjustedTime()) const; //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; + double GetChance(NodeSeconds now = AdjustedTime()) const; }; class AddrManImpl @@ -112,26 +114,26 @@ public: size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); - bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(!cs); - bool Good(const CService& addr, int64_t nTime) + bool Good(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); - void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::pair<CAddress, int64_t> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + std::pair<CAddress, NodeSeconds> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::pair<CAddress, int64_t> Select(bool newOnly) const + std::pair<CAddress, NodeSeconds> Select(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(!cs); std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(!cs); - void Connected(const CService& addr, int64_t nTime) + void Connected(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); void SetServices(const CService& addr, ServiceFlags nServices) @@ -202,7 +204,7 @@ private: int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. - int64_t nLastGood GUARDED_BY(cs){1}; + NodeSeconds m_last_good GUARDED_BY(cs){1s}; //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. std::set<int> m_tried_collisions; @@ -233,25 +235,25 @@ private: /** Attempt to add a single address to addrman's new table. * @see AddrMan::Add() for parameters. */ - bool AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool Good_(const CService& addr, bool test_before_evict, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool Add_(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool Add_(const std::vector<CAddress>& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); - std::pair<CAddress, int64_t> Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair<CAddress, NodeSeconds> Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair<CAddress, NodeSeconds> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); std::optional<AddressPosition> FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 76300f4db8..2600b03022 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -43,7 +43,7 @@ static void CreateAddresses() CAddress ret(CService(addr, port), NODE_NETWORK); - ret.nTime = GetAdjustedTime(); + ret.nTime = AdjustedTime(); return ret; }; diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 52e5cb743f..53aa470042 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -8,6 +8,7 @@ #include <chainparams.h> #include <consensus/validation.h> #include <streams.h> +#include <util/system.h> #include <validation.h> // These are the two major time-sinks which happen after we have fully received diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index eaefb9b63a..a80ec3703c 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -56,10 +56,10 @@ static void CoinSelection(benchmark::Bench& bench) addCoin(3 * COIN, wallet, wtxs); // Create coins - std::vector<COutput> coins; + wallet::CoinsResult available_coins; for (const auto& wtx : wtxs) { const auto txout = wtx->tx->vout.at(0); - coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + available_coins.bech32.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); @@ -76,7 +76,7 @@ static void CoinSelection(benchmark::Bench& bench) /*avoid_partial=*/ false, }; bench.run([&] { - auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, coins, coin_selection_params); + auto result = AttemptSelection(wallet, 1003 * COIN, filter_standard, available_coins, coin_selection_params, /*allow_mixed_output_types=*/true); assert(result); assert(result->GetSelectedValue() == 1003 * COIN); assert(result->GetInputSet().size() == 2); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index d258d7a29e..a10f7ff7d1 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -19,8 +19,6 @@ using wallet::CWallet; using wallet::DatabaseFormat; using wallet::DatabaseOptions; -using wallet::ISMINE_SPENDABLE; -using wallet::MakeWalletDatabase; using wallet::TxStateInactive; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 1817aa1a53..4656cb23e7 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -18,6 +18,7 @@ #include <consensus/validation.h> #include <core_io.h> #include <node/blockstorage.h> +#include <node/caches.h> #include <node/chainstate.h> #include <scheduler.h> #include <script/sigcache.h> @@ -78,32 +79,24 @@ int main(int argc, char* argv[]) // SETUP: Chainstate const ChainstateManager::Options chainman_opts{ - chainparams, - static_cast<int64_t(*)()>(GetTime), + .chainparams = chainparams, + .adjusted_time_callback = static_cast<int64_t (*)()>(GetTime), }; ChainstateManager chainman{chainman_opts}; - auto rv = node::LoadChainstate(false, - std::ref(chainman), - nullptr, - false, - false, - 2 << 20, - 2 << 22, - (450 << 20) - (2 << 20) - (2 << 22), - false, - false, - []() { return false; }); - if (rv.has_value()) { + node::CacheSizes cache_sizes; + cache_sizes.block_tree_db = 2 << 20; + cache_sizes.coins_db = 2 << 22; + cache_sizes.coins = (450 << 20) - (2 << 20) - (2 << 22); + node::ChainstateLoadOptions options; + options.check_interrupt = [] { return false; }; + auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options); + if (status != node::ChainstateLoadStatus::SUCCESS) { std::cerr << "Failed to load Chain state from your datadir." << std::endl; goto epilogue; } else { - auto maybe_verify_error = node::VerifyLoadedChainstate(std::ref(chainman), - false, - false, - DEFAULT_CHECKBLOCKS, - DEFAULT_CHECKLEVEL); - if (maybe_verify_error.has_value()) { + std::tie(status, error) = node::VerifyLoadedChainstate(chainman, options); + if (status != node::ChainstateLoadStatus::SUCCESS) { std::cerr << "Failed to verify loaded Chain state from your datadir." << std::endl; goto epilogue; } diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 0db2b75384..7cc956ebda 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,7 +9,7 @@ #include <chainparamsbase.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <compat/stdin.h> #include <policy/feerate.h> #include <rpc/client.h> diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e0d5c6e5dc..b006353cb0 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -8,7 +8,7 @@ #include <clientversion.h> #include <coins.h> -#include <compat.h> +#include <compat/compat.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <core_io.h> diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index 1739804edb..fb184c0486 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -11,7 +11,7 @@ #include <chainparams.h> #include <chainparamsbase.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <core_io.h> #include <streams.h> #include <util/system.h> diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7bec3292a1..a7d49452b0 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -9,7 +9,7 @@ #include <chainparams.h> #include <chainparamsbase.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <interfaces/init.h> #include <key.h> #include <logging.h> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index be894e192e..85ba88f6ab 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -9,7 +9,7 @@ #include <chainparams.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <init.h> #include <interfaces/chain.h> #include <interfaces/init.h> diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 1ad6872143..0ff79bb3ca 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -148,7 +148,7 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const const std::string& BlockFilterTypeName(BlockFilterType filter_type) { - static std::string unknown_retval = ""; + static std::string unknown_retval; auto it = g_filter_types.find(filter_type); return it != g_filter_types.end() ? it->second : unknown_retval; } diff --git a/src/chain.cpp b/src/chain.cpp index b8158f7b0b..0f898bafd5 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> +#include <tinyformat.h> #include <util/time.h> std::string CBlockFileInfo::ToString() const @@ -11,6 +12,12 @@ std::string CBlockFileInfo::ToString() const return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); } +std::string CBlockIndex::ToString() const +{ + return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)", + pprev, nHeight, hashMerkleRoot.ToString(), GetBlockHash().ToString()); +} + void CChain::SetTip(CBlockIndex *pindex) { if (pindex == nullptr) { vChain.clear(); diff --git a/src/chain.h b/src/chain.h index ecc2ae732f..627a3dfab2 100644 --- a/src/chain.h +++ b/src/chain.h @@ -11,7 +11,6 @@ #include <flatfile.h> #include <primitives/block.h> #include <sync.h> -#include <tinyformat.h> #include <uint256.h> #include <vector> @@ -263,6 +262,7 @@ public: uint256 GetBlockHash() const { + assert(phashBlock != nullptr); return *phashBlock; } @@ -301,13 +301,7 @@ public: return pbegin[(pend - pbegin) / 2]; } - std::string ToString() const - { - return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)", - pprev, nHeight, - hashMerkleRoot.ToString(), - GetBlockHash().ToString()); - } + std::string ToString() const; //! Check whether this block index entry is valid up to the passed validity level. bool IsValid(enum BlockStatus nUpTo = BLOCK_VALID_TRANSACTIONS) const @@ -402,7 +396,7 @@ public: READWRITE(obj.nNonce); } - uint256 GetBlockHash() const + uint256 ConstructBlockHash() const { CBlockHeader block; block.nVersion = nVersion; @@ -414,16 +408,8 @@ public: return block.GetHash(); } - - std::string ToString() const - { - std::string str = "CDiskBlockIndex("; - str += CBlockIndex::ToString(); - str += strprintf("\n hashBlock=%s, hashPrev=%s)", - GetBlockHash().ToString(), - hashPrev.ToString()); - return str; - } + uint256 GetBlockHash() = delete; + std::string ToString() = delete; }; /** An in-memory indexed chain of blocks. */ diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 7a7c72ea25..dd7b93234d 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -352,7 +352,7 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_TAPROOT].min_activation_height = 0; // No activation delay // message start is defined as the first 4 bytes of the sha256d of the block script - CHashWriter h(SER_DISK, 0); + HashWriter h{}; h << consensus.signet_challenge; uint256 hash = h.GetHash(); memcpy(pchMessageStart, hash.begin(), 4); diff --git a/src/compat.h b/src/compat/compat.h index 0a44b98b4e..a8e5552c0a 100644 --- a/src/compat.h +++ b/src/compat/compat.h @@ -3,21 +3,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_COMPAT_H -#define BITCOIN_COMPAT_H +#ifndef BITCOIN_COMPAT_COMPAT_H +#define BITCOIN_COMPAT_COMPAT_H #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #endif +// Windows defines FD_SETSIZE to 64 (see _fd_types.h in mingw-w64), +// which is too small for our usage, but allows us to redefine it safely. +// We redefine it to be 1024, to match glibc, see typesizes.h. #ifdef WIN32 #ifdef FD_SETSIZE -#undef FD_SETSIZE // prevent redefinition compiler warning +#undef FD_SETSIZE #endif -#define FD_SETSIZE 1024 // max number of fds in fd_set +#define FD_SETSIZE 1024 #include <winsock2.h> #include <ws2tcpip.h> -#include <stdint.h> +#include <cstdint> #else #include <fcntl.h> #include <sys/mman.h> @@ -34,49 +37,54 @@ #include <unistd.h> #endif +// We map Linux / BSD error functions and codes, to the equivalent +// Windows definitions, and use the WSA* names throughout our code. +// Note that glibc defines EWOULDBLOCK as EAGAIN (see errno.h). #ifndef WIN32 typedef unsigned int SOCKET; -#include <errno.h> +#include <cerrno> #define WSAGetLastError() errno #define WSAEINVAL EINVAL -#define WSAEALREADY EALREADY #define WSAEWOULDBLOCK EWOULDBLOCK #define WSAEAGAIN EAGAIN #define WSAEMSGSIZE EMSGSIZE #define WSAEINTR EINTR #define WSAEINPROGRESS EINPROGRESS #define WSAEADDRINUSE EADDRINUSE -#define WSAENOTSOCK EBADF #define INVALID_SOCKET (SOCKET)(~0) #define SOCKET_ERROR -1 #else -#ifndef WSAEAGAIN +// WSAEAGAIN doesn't exist on Windows #ifdef EAGAIN #define WSAEAGAIN EAGAIN #else #define WSAEAGAIN WSAEWOULDBLOCK #endif #endif -#endif +// Windows doesn't define S_IRUSR or S_IWUSR. We define both +// here, with the same values as glibc (see stat.h). #ifdef WIN32 #ifndef S_IRUSR #define S_IRUSR 0400 #define S_IWUSR 0200 #endif -#else +#endif + +// Windows defines MAX_PATH as it's maximum path length. +// We define MAX_PATH for use on non-Windows systems. +#ifndef WIN32 #define MAX_PATH 1024 #endif + +// ssize_t is POSIX, and not present when using MSVC. #ifdef _MSC_VER -#if !defined(ssize_t) -#ifdef _WIN64 -typedef int64_t ssize_t; -#else -typedef int32_t ssize_t; -#endif -#endif +#include <BaseTsd.h> +typedef SSIZE_T ssize_t; #endif +// The type of the option value passed to getsockopt & setsockopt +// differs between Windows and non-Windows. #ifndef WIN32 typedef void* sockopt_arg_type; #else @@ -119,4 +127,4 @@ bool static inline IsSelectableSocket(const SOCKET& s) { #define MSG_DONTWAIT 0 #endif -#endif // BITCOIN_COMPAT_H +#endif // BITCOIN_COMPAT_COMPAT_H diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index 57ed357645..7d14b7938e 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -298,7 +298,7 @@ void Num3072::ToBytes(unsigned char (&out)[BYTE_SIZE]) { Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { unsigned char tmp[Num3072::BYTE_SIZE]; - uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); + uint256 hashed_in{(HashWriter{} << in).GetSHA256()}; ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, Num3072::BYTE_SIZE); Num3072 out{tmp}; @@ -318,7 +318,7 @@ void MuHash3072::Finalize(uint256& out) noexcept unsigned char data[Num3072::BYTE_SIZE]; m_numerator.ToBytes(data); - out = (CHashWriter(SER_DISK, 0) << data).GetSHA256(); + out = (HashWriter{} << data).GetSHA256(); } MuHash3072& MuHash3072::operator*=(const MuHash3072& mul) noexcept diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index d4a8e4f35a..4dbc839941 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,15 +4,28 @@ #include <dbwrapper.h> -#include <memory> +#include <fs.h> +#include <logging.h> #include <random.h> +#include <tinyformat.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <algorithm> +#include <cassert> +#include <cstdarg> +#include <cstdint> +#include <cstdio> #include <leveldb/cache.h> +#include <leveldb/db.h> #include <leveldb/env.h> #include <leveldb/filter_policy.h> #include <leveldb/helpers/memenv/memenv.h> -#include <stdint.h> -#include <algorithm> +#include <leveldb/iterator.h> +#include <leveldb/options.h> +#include <leveldb/status.h> +#include <memory> +#include <optional> class CBitcoinLevelDBLogger : public leveldb::Logger { public: diff --git a/src/dbwrapper.h b/src/dbwrapper.h index cef8426d61..665eaa0e98 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -7,14 +7,26 @@ #include <clientversion.h> #include <fs.h> +#include <logging.h> #include <serialize.h> #include <span.h> #include <streams.h> -#include <util/strencodings.h> -#include <util/system.h> +#include <cstddef> +#include <cstdint> +#include <exception> #include <leveldb/db.h> +#include <leveldb/iterator.h> +#include <leveldb/options.h> +#include <leveldb/slice.h> +#include <leveldb/status.h> #include <leveldb/write_batch.h> +#include <stdexcept> +#include <string> +#include <vector> +namespace leveldb { +class Env; +} static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d125fe479b..6bab0a856c 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -49,7 +49,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS if (signer.m_fingerprint.compare(fingerprintStr) == 0) duplicate = true; } if (duplicate) break; - std::string name = ""; + std::string name; const UniValue& model_field = find_value(signer, "model"); if (model_field.isStr() && model_field.getValStr() != "") { name += model_field.getValStr(); @@ -9,6 +9,7 @@ #include <cstdio> #include <filesystem> +#include <functional> #include <iomanip> #include <ios> #include <ostream> @@ -199,6 +200,7 @@ bool create_directories(const std::filesystem::path& p, std::error_code& ec) = d /** Bridge operations to C stdio */ namespace fsbridge { + using FopenFn = std::function<FILE*(const fs::path&, const char*)>; FILE *fopen(const fs::path& p, const char *mode); /** diff --git a/src/hash.cpp b/src/hash.cpp index f58b29e3ba..111b707964 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -86,9 +86,9 @@ uint256 SHA256Uint256(const uint256& input) return result; } -CHashWriter TaggedHash(const std::string& tag) +HashWriter TaggedHash(const std::string& tag) { - CHashWriter writer(SER_GETHASH, 0); + HashWriter writer{}; uint256 taghash; CSHA256().Write((const unsigned char*)tag.data(), tag.size()).Finalize(taghash.begin()); writer << taghash << taghash; diff --git a/src/hash.h b/src/hash.h index 0ccef2105f..b1ff3acc7d 100644 --- a/src/hash.h +++ b/src/hash.h @@ -96,20 +96,12 @@ inline uint160 Hash160(const T1& in1) } /** A writer stream (for serialization) that computes a 256-bit hash. */ -class CHashWriter +class HashWriter { private: CSHA256 ctx; - const int nType; - const int nVersion; public: - - CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} - - int GetType() const { return nType; } - int GetVersion() const { return nVersion; } - void write(Span<const std::byte> src) { ctx.Write(UCharCast(src.data()), src.size()); @@ -144,6 +136,26 @@ public: return ReadLE64(result.begin()); } + template <typename T> + HashWriter& operator<<(const T& obj) + { + ::Serialize(*this, obj); + return *this; + } +}; + +class CHashWriter : public HashWriter +{ +private: + const int nType; + const int nVersion; + +public: + CHashWriter(int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) {} + + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + template<typename T> CHashWriter& operator<<(const T& obj) { // Serialize to this stream @@ -203,12 +215,12 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]); -/** Return a CHashWriter primed for tagged hashes (as specified in BIP 340). +/** Return a HashWriter primed for tagged hashes (as specified in BIP 340). * * The returned object will have SHA256(tag) written to it twice (= 64 bytes). * A tagged hash can be computed by feeding the message into this object, and - * then calling CHashWriter::GetSHA256(). + * then calling HashWriter::GetSHA256(). */ -CHashWriter TaggedHash(const std::string& tag); +HashWriter TaggedHash(const std::string& tag); #endif // BITCOIN_HASH_H diff --git a/src/httpserver.cpp b/src/httpserver.cpp index b8f69b038c..8e00a6278f 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -9,7 +9,7 @@ #include <httpserver.h> #include <chainparamsbase.h> -#include <compat.h> +#include <compat/compat.h> #include <netbase.h> #include <node/interface_ui.h> #include <rpc/protocol.h> // For HTTP status codes diff --git a/src/i2p.cpp b/src/i2p.cpp index 8611984555..c45bcc15d2 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <compat.h> +#include <compat/compat.h> #include <compat/endian.h> #include <crypto/sha256.h> #include <fs.h> @@ -5,7 +5,7 @@ #ifndef BITCOIN_I2P_H #define BITCOIN_I2P_H -#include <compat.h> +#include <compat/compat.h> #include <fs.h> #include <netaddress.h> #include <sync.h> diff --git a/src/index/base.cpp b/src/index/base.cpp index 323547900d..1ebe89ef7c 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,11 +4,15 @@ #include <chainparams.h> #include <index/base.h> +#include <interfaces/chain.h> +#include <kernel/chain.h> #include <node/blockstorage.h> +#include <node/context.h> #include <node/interface_ui.h> #include <shutdown.h> #include <tinyformat.h> #include <util/syscall_sandbox.h> +#include <util/system.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -31,6 +35,15 @@ static void FatalError(const char* fmt, const Args&... args) StartShutdown(); } +CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) +{ + CBlockLocator locator; + bool found = chain.findBlock(block_hash, interfaces::FoundBlock().locator(locator)); + assert(found); + assert(!locator.IsNull()); + return locator; +} + BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) {} @@ -49,6 +62,9 @@ void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator batch.Write(DB_BEST_BLOCK, locator); } +BaseIndex::BaseIndex(std::unique_ptr<interfaces::Chain> chain) + : m_chain{std::move(chain)} {} + BaseIndex::~BaseIndex() { Interrupt(); @@ -69,6 +85,10 @@ bool BaseIndex::Init() } else { SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); } + + // Note: this will latch to true immediately if the user starts up with an empty + // datadir and an index enabled. If this is the case, indexation will happen solely + // via `BlockConnected` signals until, possibly, the next restart. m_synced = m_best_block_index.load() == active_chain.Tip(); if (!m_synced) { bool prune_violation = false; @@ -175,12 +195,15 @@ void BaseIndex::ThreadSync() } CBlock block; + interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); if (!ReadBlockFromDisk(block, pindex, consensus_params)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; + } else { + block_info.data = █ } - if (!WriteBlock(block, pindex)) { + if (!CustomAppend(block_info)) { FatalError("%s: Failed to write block %s to index database", __func__, pindex->GetBlockHash().ToString()); return; @@ -197,22 +220,20 @@ void BaseIndex::ThreadSync() bool BaseIndex::Commit() { - CDBBatch batch(GetDB()); - if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) { - return error("%s: Failed to commit latest %s state", __func__, GetName()); - } - return true; -} - -bool BaseIndex::CommitInternal(CDBBatch& batch) -{ - LOCK(cs_main); // Don't commit anything if we haven't indexed any block yet // (this could happen if init is interrupted). - if (m_best_block_index == nullptr) { - return false; + bool ok = m_best_block_index != nullptr; + if (ok) { + CDBBatch batch(GetDB()); + ok = CustomCommit(batch); + if (ok) { + GetDB().WriteBestBlock(batch, GetLocator(*m_chain, m_best_block_index.load()->GetBlockHash())); + ok = GetDB().WriteBatch(batch); + } + } + if (!ok) { + return error("%s: Failed to commit latest %s state", __func__, GetName()); } - GetDB().WriteBestBlock(batch, m_chainstate->m_chain.GetLocator(m_best_block_index)); return true; } @@ -221,6 +242,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti assert(current_tip == m_best_block_index); assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + if (!CustomRewind({current_tip->GetBlockHash(), current_tip->nHeight}, {new_tip->GetBlockHash(), new_tip->nHeight})) { + return false; + } + // In the case of a reorg, ensure persisted block locator is not stale. // Pruning has a minimum of 288 blocks-to-keep and getting the index // out of sync may be possible but a users fault. @@ -268,8 +293,8 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const return; } } - - if (WriteBlock(*block, pindex)) { + interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex, block.get()); + if (CustomAppend(block_info)) { SetBestBlockIndex(pindex); } else { FatalError("%s: Failed to write block %s to index", @@ -346,13 +371,18 @@ void BaseIndex::Interrupt() m_interrupt(); } -bool BaseIndex::Start(CChainState& active_chainstate) +bool BaseIndex::Start() { - m_chainstate = &active_chainstate; + // m_chainstate member gives indexing code access to node internals. It is + // removed in followup https://github.com/bitcoin/bitcoin/pull/24230 + m_chainstate = &m_chain->context()->chainman->ActiveChainstate(); // Need to register this ValidationInterface before running Init(), so that // callbacks are not missed if Init sets m_synced to true. RegisterValidationInterface(this); - if (!Init()) { + if (!Init()) return false; + + const CBlockIndex* index = m_best_block_index.load(); + if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { return false; } diff --git a/src/index/base.h b/src/index/base.h index a8f6a18c8d..5a484377e7 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -6,12 +6,16 @@ #define BITCOIN_INDEX_BASE_H #include <dbwrapper.h> +#include <interfaces/chain.h> #include <threadinterrupt.h> #include <validationinterface.h> class CBlock; class CBlockIndex; class CChainState; +namespace interfaces { +class Chain; +} // namespace interfaces struct IndexSummary { std::string name; @@ -51,6 +55,10 @@ private: /// Whether the index is in sync with the main chain. The flag is flipped /// from false to true once, after which point this starts processing /// ValidationInterface notifications to stay in sync. + /// + /// Note that this will latch to true *immediately* upon startup if + /// `m_chainstate->m_chain` is empty, which will be the case upon startup + /// with an empty datadir if, e.g., `-txindex=1` is specified. std::atomic<bool> m_synced{false}; /// The last block in the chain that the index is in sync with. @@ -59,6 +67,9 @@ private: std::thread m_thread_sync; CThreadInterrupt m_interrupt; + /// Read best block locator and check that data needed to sync has not been pruned. + bool Init(); + /// Sync the index with the block index starting from the current best block. /// Intended to be run in its own thread, m_thread_sync, and can be /// interrupted with m_interrupt. Once the index gets in sync, the m_synced @@ -76,30 +87,32 @@ private: /// getting corrupted. bool Commit(); + /// Loop over disconnected blocks and call CustomRewind. + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + virtual bool AllowPrune() const = 0; protected: + std::unique_ptr<interfaces::Chain> m_chain; CChainState* m_chainstate{nullptr}; void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override; void ChainStateFlushed(const CBlockLocator& locator) override; - const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); }; - /// Initialize internal state from the database and block index. - [[nodiscard]] virtual bool Init(); + [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockKey>& block) { return true; } /// Write update index entries for a newly connected block. - virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + [[nodiscard]] virtual bool CustomAppend(const interfaces::BlockInfo& block) { return true; } /// Virtual method called internally by Commit that can be overridden to atomically /// commit more index state. - virtual bool CommitInternal(CDBBatch& batch); + virtual bool CustomCommit(CDBBatch& batch) { return true; } /// Rewind index to an earlier chain tip during a chain reorg. The tip must /// be an ancestor of the current best block. - virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + [[nodiscard]] virtual bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { return true; } virtual DB& GetDB() const = 0; @@ -110,6 +123,7 @@ protected: void SetBestBlockIndex(const CBlockIndex* block); public: + BaseIndex(std::unique_ptr<interfaces::Chain> chain); /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); @@ -124,7 +138,7 @@ public: /// Start initializes the sync state and registers the instance as a /// ValidationInterface so that it stays in sync with blockchain updates. - [[nodiscard]] bool Start(CChainState& active_chainstate); + [[nodiscard]] bool Start(); /// Stops the instance from staying in sync with blockchain updates. void Stop(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index e7fad8eb64..f4837f3456 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -9,6 +9,7 @@ #include <index/blockfilterindex.h> #include <node/blockstorage.h> #include <util/system.h> +#include <validation.h> using node::UndoReadFromDisk; @@ -94,9 +95,9 @@ struct DBHashKey { static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes; -BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, +BlockFilterIndex::BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe) - : m_filter_type(filter_type) + : BaseIndex(std::move(chain)), m_filter_type(filter_type) { const std::string& filter_name = BlockFilterTypeName(filter_type); if (filter_name.empty()) throw std::invalid_argument("unknown filter_type"); @@ -109,7 +110,7 @@ BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE); } -bool BlockFilterIndex::Init() +bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) { if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { // Check that the cause of the read failure is that the key does not exist. Any other errors @@ -124,15 +125,15 @@ bool BlockFilterIndex::Init() m_next_filter_pos.nFile = 0; m_next_filter_pos.nPos = 0; } - return BaseIndex::Init(); + return true; } -bool BlockFilterIndex::CommitInternal(CDBBatch& batch) +bool BlockFilterIndex::CustomCommit(CDBBatch& batch) { const FlatFilePos& pos = m_next_filter_pos; // Flush current filter file to disk. - CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile file{m_filter_fileseq->Open(pos)}; if (file.IsNull()) { return error("%s: Failed to open filter file %d", __func__, pos.nFile); } @@ -141,12 +142,12 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) } batch.Write(DB_FILTER_POS, pos); - return BaseIndex::CommitInternal(batch); + return true; } bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const { - CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); + AutoFile filein{m_filter_fileseq->Open(pos, true)}; if (filein.IsNull()) { return false; } @@ -178,7 +179,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { - CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile last_file{m_filter_fileseq->Open(pos)}; if (last_file.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; @@ -204,7 +205,7 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return 0; } - CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + AutoFile fileout{m_filter_fileseq->Open(pos)}; if (fileout.IsNull()) { LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); return 0; @@ -214,22 +215,25 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return data_size; } -bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; uint256 prev_header; - if (pindex->nHeight > 0) { + if (block.height > 0) { + // pindex variable gives indexing code access to node internals. It + // will be removed in upcoming commit + const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair<uint256, DBVal> read_out; - if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { return false; } - uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + uint256 expected_block_hash = *Assert(block.prev_hash); if (read_out.first != expected_block_hash) { return error("%s: previous block header belongs to unexpected block %s; expected %s", __func__, read_out.first.ToString(), expected_block_hash.ToString()); @@ -238,18 +242,18 @@ bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex prev_header = read_out.second.header; } - BlockFilter filter(m_filter_type, block, block_undo); + BlockFilter filter(m_filter_type, *Assert(block.data), block_undo); size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); if (bytes_written == 0) return false; std::pair<uint256, DBVal> value; - value.first = pindex->GetBlockHash(); + value.first = block.hash; value.second.hash = filter.GetHash(); value.second.header = filter.ComputeHeader(prev_header); value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + if (!m_db->Write(DBHeightKey(block.height), value)) { return false; } @@ -283,17 +287,15 @@ static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, return true; } -bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +bool BlockFilterIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { - assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); - CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); // During a reorg, we need to copy all filters for blocks that are getting disconnected from the // height index to the hash index so we can still find them when the height index entries are // overwritten. - if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) { return false; } @@ -303,7 +305,7 @@ bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* batch.Write(DB_FILTER_POS, m_next_filter_pos); if (!m_db->WriteBatch(batch)) return false; - return BaseIndex::Rewind(current_tip, new_tip); + return true; } static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) @@ -467,12 +469,12 @@ void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn) for (auto& entry : g_filter_indexes) fn(entry.second); } -bool InitBlockFilterIndex(BlockFilterType filter_type, +bool InitBlockFilterIndex(std::function<std::unique_ptr<interfaces::Chain>()> make_chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe) { auto result = g_filter_indexes.emplace(std::piecewise_construct, std::forward_as_tuple(filter_type), - std::forward_as_tuple(filter_type, + std::forward_as_tuple(make_chain(), filter_type, n_cache_size, f_memory, f_wipe)); return result.second; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index fef8b573e8..968eccb6b3 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -41,13 +41,13 @@ private: bool AllowPrune() const override { return true; } protected: - bool Init() override; + bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; - bool CommitInternal(CDBBatch& batch) override; + bool CustomCommit(CDBBatch& batch) override; - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; - bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; BaseIndex::DB& GetDB() const override { return *m_db; } @@ -55,7 +55,7 @@ protected: public: /** Constructs the index, which becomes available to be queried. */ - explicit BlockFilterIndex(BlockFilterType filter_type, + explicit BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); BlockFilterType GetFilterType() const { return m_filter_type; } @@ -88,7 +88,7 @@ void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn); * Initialize a block filter index for the given type if one does not already exist. Returns true if * a new index is created and false if one has already been initialized. */ -bool InitBlockFilterIndex(BlockFilterType filter_type, +bool InitBlockFilterIndex(std::function<std::unique_ptr<interfaces::Chain>()> make_chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); /** diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 687e330fe0..b9029e946a 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -10,6 +10,7 @@ #include <serialize.h> #include <txdb.h> #include <undo.h> +#include <util/system.h> #include <validation.h> using kernel::CCoinsStats; @@ -102,7 +103,8 @@ struct DBHashKey { std::unique_ptr<CoinStatsIndex> g_coin_stats_index; -CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) +CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe) + : BaseIndex(std::move(chain)) { fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"}; fs::create_directories(path); @@ -110,24 +112,27 @@ CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); } -bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; - const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())}; m_total_subsidy += block_subsidy; // Ignore genesis block - if (pindex->nHeight > 0) { + if (block.height > 0) { + // pindex variable gives indexing code access to node internals. It + // will be removed in upcoming commit + const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair<uint256, DBVal> read_out; - if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { return false; } - uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + uint256 expected_block_hash{*Assert(block.prev_hash)}; if (read_out.first != expected_block_hash) { LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", read_out.first.ToString(), expected_block_hash.ToString()); @@ -139,12 +144,13 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) } // TODO: Deduplicate BIP30 related code - bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || - (pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; + bool is_bip30_block{(block.height == 91722 && block.hash == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (block.height == 91812 && block.hash == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; // Add the new utxos created from the block - for (size_t i = 0; i < block.vtx.size(); ++i) { - const auto& tx{block.vtx.at(i)}; + assert(block.data); + for (size_t i = 0; i < block.data->vtx.size(); ++i) { + const auto& tx{block.data->vtx.at(i)}; // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { @@ -155,7 +161,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; - Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + Coin coin{out, block.height, tx->IsCoinBase()}; COutPoint outpoint{tx->GetHash(), j}; // Skip unspendable coins @@ -211,7 +217,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_total_unspendables_unclaimed_rewards += unclaimed_rewards; std::pair<uint256, DBVal> value; - value.first = pindex->GetBlockHash(); + value.first = block.hash; value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; @@ -231,7 +237,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Intentionally do not update DB_MUHASH here so it stays in sync with // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown. - return m_db->Write(DBHeightKey(pindex->nHeight), value); + return m_db->Write(DBHeightKey(block.height), value); } static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, @@ -260,17 +266,15 @@ static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, return true; } -bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { - assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); - CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); // During a reorg, we need to copy all hash digests for blocks that are // getting disconnected from the height index to the hash index so we can // still find them when the height index entries are overwritten. - if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) { return false; } @@ -278,7 +282,8 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n { LOCK(cs_main); - const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)}; + const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)}; const auto& consensus_params{Params().GetConsensus()}; do { @@ -292,29 +297,29 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n ReverseBlock(block, iter_tip); iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); - } while (new_tip != iter_tip); + } while (new_tip_index != iter_tip); } - return BaseIndex::Rewind(current_tip, new_tip); + return true; } -static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockKey& block, DBVal& result) { // First check if the result is stored under the height index and the value // there matches the block hash. This should be the case if the block is on // the active chain. std::pair<uint256, DBVal> read_out; - if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + if (!db.Read(DBHeightKey(block.height), read_out)) { return false; } - if (read_out.first == block_index->GetBlockHash()) { + if (read_out.first == block.hash) { result = std::move(read_out.second); return true; } // If value at the height index corresponds to an different block, the // result will be stored in the hash index. - return db.Read(DBHashKey(block_index->GetBlockHash()), result); + return db.Read(DBHashKey(block.hash), result); } std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_index) const @@ -323,7 +328,7 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_ stats.index_used = true; DBVal entry; - if (!LookUpOne(*m_db, block_index, entry)) { + if (!LookUpOne(*m_db, {block_index->GetBlockHash(), block_index->nHeight}, entry)) { return std::nullopt; } @@ -344,7 +349,7 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_ return stats; } -bool CoinStatsIndex::Init() +bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) { if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not @@ -356,13 +361,9 @@ bool CoinStatsIndex::Init() } } - if (!BaseIndex::Init()) return false; - - const CBlockIndex* pindex{CurrentIndex()}; - - if (pindex) { + if (block) { DBVal entry; - if (!LookUpOne(*m_db, pindex, entry)) { + if (!LookUpOne(*m_db, *block, entry)) { return error("%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } @@ -391,12 +392,12 @@ bool CoinStatsIndex::Init() return true; } -bool CoinStatsIndex::CommitInternal(CDBBatch& batch) +bool CoinStatsIndex::CustomCommit(CDBBatch& batch) { // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK // to prevent an inconsistent state of the DB. batch.Write(DB_MUHASH, m_muhash); - return BaseIndex::CommitInternal(batch); + return true; } // Reverse a single block as part of a reorg diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index cae052d913..c4af223388 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -39,13 +39,13 @@ private: bool AllowPrune() const override { return true; } protected: - bool Init() override; + bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; - bool CommitInternal(CDBBatch& batch) override; + bool CustomCommit(CDBBatch& batch) override; - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; - bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; BaseIndex::DB& GetDB() const override { return *m_db; } @@ -53,7 +53,7 @@ protected: public: // Constructs the index, which becomes available to be queried. - explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + explicit CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Look up stats for a specific block using CBlockIndex std::optional<kernel::CCoinsStats> LookUpStats(const CBlockIndex* block_index) const; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 97c11c4383..b719aface8 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -48,23 +48,22 @@ bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_ return WriteBatch(batch); } -TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) - : m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) +TxIndex::TxIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe) + : BaseIndex(std::move(chain)), m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {} TxIndex::~TxIndex() = default; -bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) { // Exclude genesis block transaction because outputs are not spendable. - if (pindex->nHeight == 0) return true; + if (block.height == 0) return true; - CDiskTxPos pos{ - WITH_LOCK(::cs_main, return pindex->GetBlockPos()), - GetSizeOfCompactSize(block.vtx.size())}; + assert(block.data); + CDiskTxPos pos({block.file_number, block.data_pos}, GetSizeOfCompactSize(block.data->vtx.size())); std::vector<std::pair<uint256, CDiskTxPos>> vPos; - vPos.reserve(block.vtx.size()); - for (const auto& tx : block.vtx) { + vPos.reserve(block.data->vtx.size()); + for (const auto& tx : block.data->vtx) { vPos.emplace_back(tx->GetHash(), pos); pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); } diff --git a/src/index/txindex.h b/src/index/txindex.h index ec339abaa1..be240c4582 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -23,7 +23,7 @@ private: bool AllowPrune() const override { return false; } protected: - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; BaseIndex::DB& GetDB() const override; @@ -31,7 +31,7 @@ protected: public: /// Constructs the index, which becomes available to be queried. - explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + explicit TxIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Destructor is declared because this class contains a unique_ptr to an incomplete type. virtual ~TxIndex() override; diff --git a/src/init.cpp b/src/init.cpp index 816404a84f..a94bbe6460 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,6 +10,7 @@ #include <init.h> #include <kernel/checks.h> +#include <kernel/mempool_persist.h> #include <addrman.h> #include <banman.h> @@ -41,6 +42,7 @@ #include <node/chainstate.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/mempool_persist_args.h> #include <node/miner.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -102,14 +104,16 @@ #include <zmq/zmqrpc.h> #endif +using kernel::DumpMempool; + using node::CacheSizes; using node::CalculateCacheSizes; -using node::ChainstateLoadVerifyError; -using node::ChainstateLoadingError; -using node::CleanupBlockRevFiles; +using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPAFTERBLOCKIMPORT; using node::LoadChainstate; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; using node::ThreadImport; using node::VerifyLoadedChainstate; @@ -245,8 +249,8 @@ void Shutdown(NodeContext& node) node.addrman.reset(); node.netgroupman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(*node.mempool); + if (node.mempool && node.mempool->GetLoadTried() && ShouldPersistMempool(*node.args)) { + DumpMempool(*node.mempool, MempoolPath(*node.args)); } // Drop transactions we were still watching, and record fee estimations. @@ -1092,21 +1096,8 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - auto maybe_error = kernel::SanityChecks(kernel); - - if (maybe_error.has_value()) { - switch (maybe_error.value()) { - case kernel::SanityCheckError::ERROR_ECC: - InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); - break; - case kernel::SanityCheckError::ERROR_RANDOM: - InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); - break; - case kernel::SanityCheckError::ERROR_CHRONO: - InitError(Untranslated("Clock epoch mismatch. Aborting.")); - break; - } // no default case, so the compiler can warn about missing cases - + if (auto error = kernel::SanityChecks(kernel)) { + InitError(*error); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1440,118 +1431,60 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.mempool = std::make_unique<CTxMemPool>(mempool_opts); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; node.chainman = std::make_unique<ChainstateManager>(chainman_opts); ChainstateManager& chainman = *node.chainman; - const bool fReset = fReindex; - bilingual_str strLoadError; + node::ChainstateLoadOptions options; + options.mempool = Assert(node.mempool.get()); + options.reindex = node::fReindex; + options.reindex_chainstate = fReindexChainState; + options.prune = node::fPruneMode; + options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + options.check_interrupt = ShutdownRequested; + options.coins_error_cb = [] { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down."), + "", CClientUIInterface::MSG_ERROR); + }; uiInterface.InitMessage(_("Loading block index…").translated); const int64_t load_block_index_start_time = GetTimeMillis(); - std::optional<ChainstateLoadingError> maybe_load_error; - try { - maybe_load_error = LoadChainstate(fReset, - chainman, - Assert(node.mempool.get()), - fPruneMode, - fReindexChainState, - cache_sizes.block_tree_db, - cache_sizes.coins_db, - cache_sizes.coins, - /*block_tree_db_in_memory=*/false, - /*coins_db_in_memory=*/false, - /*shutdown_requested=*/ShutdownRequested, - /*coins_error_cb=*/[]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), - "", CClientUIInterface::MSG_ERROR); - }); - } catch (const std::exception& e) { - LogPrintf("%s\n", e.what()); - maybe_load_error = ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED; - } - if (maybe_load_error.has_value()) { - switch (maybe_load_error.value()) { - case ChainstateLoadingError::ERROR_LOADING_BLOCK_DB: - strLoadError = _("Error loading block database"); - break; - case ChainstateLoadingError::ERROR_BAD_GENESIS_BLOCK: - // If the loaded chain has a wrong genesis, bail out immediately - // (we're likely using a testnet datadir, or the other way around). - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); - case ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX: - strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); - break; - case ChainstateLoadingError::ERROR_LOAD_GENESIS_BLOCK_FAILED: - strLoadError = _("Error initializing block database"); - break; - case ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED: - return InitError(_("Unsupported chainstate database format found. " - "Please restart with -reindex-chainstate. This will " - "rebuild the chainstate database.")); - case ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED: - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); - break; - case ChainstateLoadingError::ERROR_LOADCHAINTIP_FAILED: - strLoadError = _("Error initializing block database"); - break; - case ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED: - strLoadError = _("Error opening block database"); - break; - case ChainstateLoadingError::ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED: - strLoadError = strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), - chainman.GetConsensus().SegwitHeight); - break; - case ChainstateLoadingError::SHUTDOWN_PROBED: - break; - } - } else { - std::optional<ChainstateLoadVerifyError> maybe_verify_error; + auto catch_exceptions = [](auto&& f) { try { - uiInterface.InitMessage(_("Verifying blocks…").translated); - auto check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); - if (chainman.m_blockman.m_have_pruned && check_blocks > MIN_BLOCKS_TO_KEEP) { - LogPrintfCategory(BCLog::PRUNE, "pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - maybe_verify_error = VerifyLoadedChainstate(chainman, - fReset, - fReindexChainState, - check_blocks, - args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL)); + return f(); } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - maybe_verify_error = ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE; + return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); } - if (maybe_verify_error.has_value()) { - switch (maybe_verify_error.value()) { - case ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE: - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct"); - break; - case ChainstateLoadVerifyError::ERROR_CORRUPTED_BLOCK_DB: - strLoadError = _("Corrupted block database detected"); - break; - case ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE: - strLoadError = _("Error opening block database"); - break; - } - } else { + }; + auto [status, error] = catch_exceptions([&]{ return LoadChainstate(chainman, cache_sizes, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + uiInterface.InitMessage(_("Verifying blocks…").translated); + if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { + LogPrintfCategory(BCLog::PRUNE, "pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + std::tie(status, error) = catch_exceptions([&]{ return VerifyLoadedChainstate(chainman, options);}); + if (status == node::ChainstateLoadStatus::SUCCESS) { fLoaded = true; LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); } } + if (status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB) { + return InitError(error); + } + if (!fLoaded && !ShutdownRequested()) { // first suggest a reindex - if (!fReset) { + if (!options.reindex) { bool fRet = uiInterface.ThreadSafeQuestion( - strLoadError + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), - strLoadError.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), + error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; @@ -1561,7 +1494,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return false; } } else { - return InitError(strLoadError); + return InitError(error); } } } @@ -1587,22 +1520,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(*error); } - g_txindex = std::make_unique<TxIndex>(cache_sizes.tx_index, false, fReindex); - if (!g_txindex->Start(chainman.ActiveChainstate())) { + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); + if (!g_txindex->Start()) { return false; } } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex(filter_type, cache_sizes.filter_index, false, fReindex); - if (!GetBlockFilterIndex(filter_type)->Start(chainman.ActiveChainstate())) { + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, fReindex); + if (!GetBlockFilterIndex(filter_type)->Start()) { return false; } } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(/* cache size */ 0, false, fReindex); - if (!g_coin_stats_index->Start(chainman.ActiveChainstate())) { + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /* cache size */ 0, false, fReindex); + if (!g_coin_stats_index->Start()) { return false; } } @@ -1669,7 +1602,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args); + ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index df9e55874f..5fc0e540a9 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -18,6 +18,7 @@ class ArgsManager; class CBlock; +class CBlockUndo; class CFeeRate; class CRPCCommand; class CScheduler; @@ -37,6 +38,12 @@ namespace interfaces { class Handler; class Wallet; +//! Hash/height pair to help track and identify blocks. +struct BlockKey { + uint256 hash; + int height = -1; +}; + //! Helper for findBlock to selectively return pieces of block data. If block is //! found, data will be returned by setting specified output variables. If block //! is not found, output variables will keep their previous values. @@ -50,6 +57,8 @@ public: FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } //! Return whether block is in the active (most-work) chain. FoundBlock& inActiveChain(bool& in_active_chain) { m_in_active_chain = &in_active_chain; return *this; } + //! Return locator if block is in the active chain. + FoundBlock& locator(CBlockLocator& locator) { m_locator = &locator; return *this; } //! Return next block in the active chain if current block is in the active chain. FoundBlock& nextBlock(const FoundBlock& next_block) { m_next_block = &next_block; return *this; } //! Read block data from disk. If the block exists but doesn't have data @@ -62,11 +71,25 @@ public: int64_t* m_max_time = nullptr; int64_t* m_mtp_time = nullptr; bool* m_in_active_chain = nullptr; + CBlockLocator* m_locator = nullptr; const FoundBlock* m_next_block = nullptr; CBlock* m_data = nullptr; mutable bool found = false; }; +//! Block data sent with blockConnected, blockDisconnected notifications. +struct BlockInfo { + const uint256& hash; + const uint256* prev_hash = nullptr; + int height = -1; + int file_number = -1; + unsigned data_pos = 0; + const CBlock* data = nullptr; + const CBlockUndo* undo_data = nullptr; + + BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} +}; + //! Interface giving clients (wallet processes, maybe other analysis tools in //! the future) ability to access to the chain state, receive notifications, //! estimate fees, and submit transactions. @@ -239,8 +262,8 @@ public: virtual ~Notifications() {} virtual void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) {} virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) {} - virtual void blockConnected(const CBlock& block, int height) {} - virtual void blockDisconnected(const CBlock& block, int height) {} + virtual void blockConnected(const BlockInfo& block) {} + virtual void blockDisconnected(const BlockInfo& block) {} virtual void updatedBlockTip() {} virtual void chainStateFlushed(const CBlockLocator& locator) {} }; @@ -287,6 +310,13 @@ public: //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; + + //! Return true if an assumed-valid chain is in use. + virtual bool hasAssumedValidChain() = 0; + + //! Get internal node context. Useful for testing, but not + //! accessible across processes. + virtual node::NodeContext* context() { return nullptr; } }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index fe198c999b..e29fefae56 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -329,7 +329,7 @@ public: virtual std::string getWalletDir() = 0; //! Restore backup wallet - virtual std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + virtual BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) = 0; //! Return available wallets in wallet directory. virtual std::vector<std::string> listWalletDir() = 0; diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp new file mode 100644 index 0000000000..82e77125d7 --- /dev/null +++ b/src/kernel/chain.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chain.h> +#include <interfaces/chain.h> +#include <sync.h> +#include <uint256.h> + +class CBlock; + +namespace kernel { +interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data) +{ + interfaces::BlockInfo info{index ? *index->phashBlock : uint256::ZERO}; + if (index) { + info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; + info.height = index->nHeight; + LOCK(::cs_main); + info.file_number = index->nFile; + info.data_pos = index->nDataPos; + } + info.data = data; + return info; +} +} // namespace kernel diff --git a/src/kernel/chain.h b/src/kernel/chain.h new file mode 100644 index 0000000000..f0750f8266 --- /dev/null +++ b/src/kernel/chain.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_CHAIN_H +#define BITCOIN_KERNEL_CHAIN_H + +class CBlock; +class CBlockIndex; +namespace interfaces { +struct BlockInfo; +} // namespace interfaces + +namespace kernel { +//! Return data from block index. +interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* block_index, const CBlock* data = nullptr); +} // namespace kernel + +#endif // BITCOIN_KERNEL_CHAIN_H diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 575d94e2e9..510a1f9edc 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -10,6 +10,8 @@ class CChainParams; +namespace kernel { + /** * An options struct for `ChainstateManager`, more ergonomically referred to as * `ChainstateManager::Options` due to the using-declaration in @@ -20,4 +22,6 @@ struct ChainstateManagerOpts { const std::function<int64_t()> adjusted_time_callback{nullptr}; }; +} // namespace kernel + #endif // BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 2a1dd3bfa2..4c303c172c 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -7,21 +7,24 @@ #include <key.h> #include <random.h> #include <util/time.h> +#include <util/translation.h> + +#include <memory> namespace kernel { -std::optional<SanityCheckError> SanityChecks(const Context&) +std::optional<bilingual_str> SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return SanityCheckError::ERROR_ECC; + return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); } if (!Random_SanityCheck()) { - return SanityCheckError::ERROR_RANDOM; + return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); } if (!ChronoSanityCheck()) { - return SanityCheckError::ERROR_CHRONO; + return Untranslated("Clock epoch mismatch. Aborting."); } return std::nullopt; diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 80b207f607..3eb14824fb 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -7,20 +7,16 @@ #include <optional> +struct bilingual_str; + namespace kernel { struct Context; -enum class SanityCheckError { - ERROR_ECC, - ERROR_RANDOM, - ERROR_CHRONO, -}; - /** * Ensure a usable environment with all necessary library support. */ -std::optional<SanityCheckError> SanityChecks(const Context&); +std::optional<bilingual_str> SanityChecks(const Context&); } diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index f380871627..06a4b8c974 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -4,16 +4,32 @@ #include <kernel/coinstats.h> +#include <chain.h> #include <coins.h> #include <crypto/muhash.h> #include <hash.h> +#include <node/blockstorage.h> +#include <primitives/transaction.h> +#include <script/script.h> #include <serialize.h> +#include <span.h> +#include <streams.h> +#include <sync.h> +#include <tinyformat.h> #include <uint256.h> +#include <util/check.h> #include <util/overflow.h> #include <util/system.h> #include <validation.h> +#include <version.h> +#include <cassert> +#include <iosfwd> +#include <iterator> #include <map> +#include <memory> +#include <string> +#include <utility> namespace kernel { @@ -52,7 +68,7 @@ CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) { //! It is also possible, though very unlikely, that a change in this //! construction could cause a previously invalid (and potentially malicious) //! UTXO snapshot to be considered valid. -static void ApplyHash(CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(HashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { for (auto it = outputs.begin(); it != outputs.end(); ++it) { if (it == outputs.begin()) { @@ -143,7 +159,7 @@ std::optional<CCoinsStats> ComputeUTXOStats(CoinStatsHashType hash_type, CCoinsV bool success = [&]() -> bool { switch (hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + HashWriter ss{}; return ComputeUTXOStats(view, stats, ss, interruption_point); } case(CoinStatsHashType::MUHASH): { @@ -164,7 +180,7 @@ std::optional<CCoinsStats> ComputeUTXOStats(CoinStatsHashType hash_type, CCoinsV } // The legacy hash serializes the hashBlock -static void PrepareHash(CHashWriter& ss, const CCoinsStats& stats) +static void PrepareHash(HashWriter& ss, const CCoinsStats& stats) { ss << stats.hashBlock; } @@ -172,7 +188,7 @@ static void PrepareHash(CHashWriter& ss, const CCoinsStats& stats) static void PrepareHash(MuHash3072& muhash, CCoinsStats& stats) {} static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} -static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) +static void FinalizeHash(HashWriter& ss, CCoinsStats& stats) { stats.hashSerialized = ss.GetHash(); } diff --git a/src/kernel/coinstats.h b/src/kernel/coinstats.h index a15957233f..b7c1328e93 100644 --- a/src/kernel/coinstats.h +++ b/src/kernel/coinstats.h @@ -5,16 +5,18 @@ #ifndef BITCOIN_KERNEL_COINSTATS_H #define BITCOIN_KERNEL_COINSTATS_H -#include <chain.h> -#include <coins.h> #include <consensus/amount.h> #include <streams.h> #include <uint256.h> #include <cstdint> #include <functional> +#include <optional> class CCoinsView; +class Coin; +class COutPoint; +class CScript; namespace node { class BlockManager; } // namespace node diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp new file mode 100644 index 0000000000..1a1cf2bbdc --- /dev/null +++ b/src/kernel/mempool_persist.cpp @@ -0,0 +1,189 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <kernel/mempool_persist.h> + +#include <clientversion.h> +#include <consensus/amount.h> +#include <fs.h> +#include <logging.h> +#include <primitives/transaction.h> +#include <serialize.h> +#include <shutdown.h> +#include <streams.h> +#include <sync.h> +#include <txmempool.h> +#include <uint256.h> +#include <util/system.h> +#include <util/time.h> +#include <validation.h> + +#include <chrono> +#include <cstdint> +#include <cstdio> +#include <exception> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <stdexcept> +#include <utility> +#include <vector> + +using fsbridge::FopenFn; + +namespace kernel { + +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, CChainState& active_chainstate, FopenFn mockable_fopen_function) +{ + if (load_path.empty()) return false; + + FILE* filestr{mockable_fopen_function(load_path, "rb")}; + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; + auto now = NodeClock::now(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + while (num) { + --num; + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); + } + if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) { + LOCK(cs_main); + const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); + if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (ShutdownRequested()) + return false; + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } + + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + return true; +} + +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) +{ + auto start = SteadyClock::now(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); + + { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); + } + + auto mid = SteadyClock::now(); + + try { + FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; + if (!filestr) { + return false; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!skip_file_commit && !FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + if (!RenameOver(dump_path + ".new", dump_path)) { + throw std::runtime_error("Rename failed"); + } + auto last = SteadyClock::now(); + + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", + Ticks<SecondsDouble>(mid - start), + Ticks<SecondsDouble>(last - mid)); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + +} // namespace kernel diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h new file mode 100644 index 0000000000..9a15ec6dca --- /dev/null +++ b/src/kernel/mempool_persist.h @@ -0,0 +1,28 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H + +#include <fs.h> + +class CChainState; +class CTxMemPool; + +namespace kernel { + +/** Dump the mempool to disk. */ +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, + bool skip_file_commit = false); + +/** Load the mempool from disk. */ +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, + CChainState& active_chainstate, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + +} // namespace kernel + + +#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index 18626b327c..fac41be6ce 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -49,7 +49,7 @@ constexpr const int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 4096 : 0; int g_mmap_limit = kDefaultMmapLimit; // Common flags defined for all posix open operations -#if defined(HAVE_O_CLOEXEC) +#if HAVE_O_CLOEXEC constexpr const int kOpenBaseFlags = O_CLOEXEC; #else constexpr const int kOpenBaseFlags = 0; diff --git a/src/mapport.cpp b/src/mapport.cpp index 235e6f904c..6262e51879 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -19,7 +19,7 @@ #include <util/thread.h> #ifdef USE_NATPMP -#include <compat.h> +#include <compat/compat.h> #include <natpmp.h> #endif // USE_NATPMP diff --git a/src/net.cpp b/src/net.cpp index 90c73d583e..c4aaac4986 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -13,7 +13,7 @@ #include <addrman.h> #include <banman.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> #include <node/eviction.h> @@ -85,8 +85,8 @@ static constexpr int DNSSEEDS_DELAY_PEER_THRESHOLD = 1000; // "many" vs "few" pe /** The default timeframe for -maxuploadtarget. 1 day. */ static constexpr std::chrono::seconds MAX_UPLOAD_TIMEFRAME{60 * 60 * 24}; -// We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. -#define FEELER_SLEEP_WINDOW 1 +// A random time period (0 to 1 seconds) is added to feeler connections to prevent synchronization. +static constexpr auto FEELER_SLEEP_WINDOW{1s}; /** Used to pass flags to the Bind() function */ enum BindFlags { @@ -187,7 +187,7 @@ static std::vector<CAddress> ConvertSeeds(const std::vector<uint8_t> &vSeedsIn) // it'll get a pile of addresses with newer timestamps. // Seed nodes are given a random 'last seen time' of between one and two // weeks ago. - const int64_t nOneWeek = 7*24*60*60; + const auto one_week{7 * 24h}; std::vector<CAddress> vSeedsOut; FastRandomContext rng; CDataStream s(vSeedsIn, SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); @@ -195,7 +195,7 @@ static std::vector<CAddress> ConvertSeeds(const std::vector<uint8_t> &vSeedsIn) CService endpoint; s >> endpoint; CAddress addr{endpoint, GetDesirableServiceFlags(NODE_NONE)}; - addr.nTime = GetTime() - rng.randrange(nOneWeek) - nOneWeek; + addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - one_week, -one_week); LogPrint(BCLog::NET, "Added hardcoded seed: %s\n", addr.ToString()); vSeedsOut.push_back(addr); } @@ -206,15 +206,13 @@ static std::vector<CAddress> ConvertSeeds(const std::vector<uint8_t> &vSeedsIn) // Otherwise, return the unroutable 0.0.0.0 but filled in with // the normal parameters, since the IP may be changed to a useful // one by discovery. -CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) +CService GetLocalAddress(const CNetAddr& addrPeer) { - CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); + CService ret{CNetAddr(), GetListenPort()}; CService addr; - if (GetLocal(addr, paddrPeer)) - { - ret = CAddress(addr, nLocalServices); + if (GetLocal(addr, &addrPeer)) { + ret = CService{addr}; } - ret.nTime = GetAdjustedTime(); return ret; } @@ -233,35 +231,35 @@ bool IsPeerAddrLocalGood(CNode *pnode) IsReachable(addrLocal.GetNetwork()); } -std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode) +std::optional<CService> GetLocalAddrForPeer(CNode& node) { - CAddress addrLocal = GetLocalAddress(&pnode->addr, pnode->GetLocalServices()); + CService addrLocal{GetLocalAddress(node.addr)}; if (gArgs.GetBoolArg("-addrmantest", false)) { // use IPv4 loopback during addrmantest - addrLocal = CAddress(CService(LookupNumeric("127.0.0.1", GetListenPort())), pnode->GetLocalServices()); + addrLocal = CService(LookupNumeric("127.0.0.1", GetListenPort())); } // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. FastRandomContext rng; - if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || + if (IsPeerAddrLocalGood(&node) && (!addrLocal.IsRoutable() || rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { - if (pnode->IsInboundConn()) { + if (node.IsInboundConn()) { // For inbound connections, assume both the address and the port // as seen from the peer. - addrLocal = CAddress{pnode->GetAddrLocal(), addrLocal.nServices, addrLocal.nTime}; + addrLocal = CService{node.GetAddrLocal()}; } else { // For outbound connections, assume just the address as seen from // the peer and leave the port in `addrLocal` as returned by // `GetLocalAddress()` above. The peer has no way to observe our // listening port when we have initiated the connection. - addrLocal.SetIP(pnode->GetAddrLocal()); + addrLocal.SetIP(node.GetAddrLocal()); } } if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) { - LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToString(), pnode->GetId()); + LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToString(), node.GetId()); return addrLocal; } // Address is unroutable. Don't advertise. @@ -454,10 +452,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo } } - /// debug print LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "trying connection %s lastseen=%.1fhrs\n", - pszDest ? pszDest : addrConnect.ToString(), - pszDest ? 0.0 : (double)(GetAdjustedTime() - addrConnect.nTime) / 3600.0); + pszDest ? pszDest : addrConnect.ToString(), + Ticks<HoursDouble>(pszDest ? 0h : AdjustedTime() - addrConnect.nTime)); // Resolve const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : @@ -543,7 +540,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addr_bind = GetBindAddress(*sock); } CNode* pnode = new CNode(id, - nLocalServices, std::move(sock), addrConnect, CalculateKeyedNetGroup(addrConnect), @@ -603,7 +599,6 @@ Network CNode::ConnectedThroughNetwork() const void CNode::CopyStats(CNodeStats& stats) { stats.nodeid = this->GetId(); - X(nServices); X(addr); X(addrBind); stats.m_network = ConnectedThroughNetwork(); @@ -880,7 +875,7 @@ bool CConnman::AttemptToEvictConnection() .m_min_ping_time = node->m_min_ping_time, .m_last_block_time = node->m_last_block_time, .m_last_tx_time = node->m_last_tx_time, - .fRelevantServices = HasAllDesirableServiceFlags(node->nServices), + .fRelevantServices = node->m_has_all_wanted_services, .m_relay_txs = node->m_relays_txs.load(), .fBloomFilter = node->m_bloom_filter_loaded.load(), .nKeyedNetGroup = node->nKeyedNetGroup, @@ -1014,7 +1009,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); CNode* pnode = new CNode(id, - nodeServices, std::move(sock), addr, CalculateKeyedNetGroup(addr), @@ -1026,7 +1020,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, pnode->AddRef(); pnode->m_permissionFlags = permissionFlags; pnode->m_prefer_evict = discouraged; - m_msgproc->InitializeNode(pnode); + m_msgproc->InitializeNode(*pnode, nodeServices); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -1474,9 +1468,8 @@ void CConnman::ThreadDNSAddressSeed() unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed if (LookupHost(host, vIPs, nMaxIPs, true)) { for (const CNetAddr& ip : vIPs) { - int nOneDay = 24*3600; CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); - addr.nTime = GetTime() - 3*nOneDay - rng.randrange(4*nOneDay); // use a random age between 3 and 7 days old + addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old vAdd.push_back(addr); found++; } @@ -1573,6 +1566,7 @@ int CConnman::GetExtraBlockRelayCount() const void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); + FastRandomContext rng; // Connect to specific addresses if (!connect.empty()) { @@ -1741,7 +1735,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) addrman.ResolveCollisions(); - int64_t nANow = GetAdjustedTime(); + const auto nANow{AdjustedTime()}; int nTries = 0; while (!interruptNet) { @@ -1764,7 +1758,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) break; CAddress addr; - int64_t addr_last_try{0}; + NodeSeconds addr_last_try{0s}; if (fFeeler) { // First, try to get a tried table collision address. This returns @@ -1804,8 +1798,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; // only consider very recently tried nodes after 30 failed attempts - if (nANow - addr_last_try < 600 && nTries < 30) + if (nANow - addr_last_try < 10min && nTries < 30) { continue; + } // for non-feelers, require all the services we'll want, // for feelers, only require they be a full node (only because most @@ -1826,12 +1821,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } if (addrConnect.IsValid()) { - if (fFeeler) { // Add small amount of random noise before connection to avoid synchronization. - int randsleep = GetRand<int>(FEELER_SLEEP_WINDOW * 1000); - if (!interruptNet.sleep_for(std::chrono::milliseconds(randsleep))) + if (!interruptNet.sleep_for(rng.rand_uniform_duration<CThreadInterrupt::Clock>(FEELER_SLEEP_WINDOW))) { return; + } LogPrint(BCLog::NET, "Making feeler connection to %s\n", addrConnect.ToString()); } @@ -1964,7 +1958,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai if (grantOutbound) grantOutbound->MoveTo(pnode->grantOutbound); - m_msgproc->InitializeNode(pnode); + m_msgproc->InitializeNode(*pnode, nLocalServices); { LOCK(m_nodes_mutex); m_nodes.push_back(pnode); @@ -2708,7 +2702,10 @@ ServiceFlags CConnman::GetLocalServices() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion) +CNode::CNode(NodeId idIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, + uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, + const CAddress& addrBindIn, const std::string& addrNameIn, + ConnectionType conn_type_in, bool inbound_onion) : m_sock{sock}, m_connected{GetTime<std::chrono::seconds>()}, addr(addrIn), @@ -2718,8 +2715,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> s nKeyedNetGroup(nKeyedNetGroupIn), id(idIn), nLocalHostNonce(nLocalHostNonceIn), - m_conn_type(conn_type_in), - nLocalServices(nLocalServicesIn) + m_conn_type(conn_type_in) { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); @@ -2828,7 +2824,7 @@ void CaptureMessageToFile(const CAddress& addr, fs::create_directories(base_path); fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); - CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + AutoFile f{fsbridge::fopen(path, "ab")}; ser_writedata64(f, now.count()); f.write(MakeByteSpan(msg_type)); @@ -8,7 +8,7 @@ #include <chainparams.h> #include <common/bloom.h> -#include <compat.h> +#include <compat/compat.h> #include <node/connection_types.h> #include <consensus/amount.h> #include <crypto/siphash.h> @@ -144,8 +144,8 @@ enum }; bool IsPeerAddrLocalGood(CNode *pnode); -/** Returns a local address that we should advertise to this peer */ -std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode); +/** Returns a local address that we should advertise to this peer. */ +std::optional<CService> GetLocalAddrForPeer(CNode& node); /** * Mark a network as reachable or unreachable (no automatic connects to it) @@ -163,7 +163,7 @@ void RemoveLocal(const CService& addr); bool SeenLocal(const CService& addr); bool IsLocal(const CService& addr); bool GetLocal(CService &addr, const CNetAddr *paddrPeer = nullptr); -CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices); +CService GetLocalAddress(const CNetAddr& addrPeer); extern bool fDiscover; @@ -187,7 +187,6 @@ class CNodeStats { public: NodeId nodeid; - ServiceFlags nServices; std::chrono::seconds m_last_send; std::chrono::seconds m_last_recv; std::chrono::seconds m_last_tx_time; @@ -346,7 +345,6 @@ public: std::unique_ptr<TransportSerializer> m_serializer; NetPermissionFlags m_permissionFlags{NetPermissionFlags::None}; - std::atomic<ServiceFlags> nServices{NODE_NONE}; /** * Socket used for communication with the node. @@ -399,8 +397,6 @@ public: bool HasPermission(NetPermissionFlags permission) const { return NetPermissions::HasFlag(m_permissionFlags, permission); } - bool fClient{false}; // set by version message - bool m_limited_node{false}; //after BIP159, set by version message /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the @@ -484,6 +480,9 @@ public: // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) std::atomic<bool> m_bip152_highbandwidth_from{false}; + /** Whether this peer provides all services that we want. Used for eviction decisions */ + std::atomic_bool m_has_all_wanted_services{false}; + /** Whether we should relay transactions to this peer (their version * message did not include fRelay=false and this is not a block-relay-only * connection). This only changes from false to true. It will never change @@ -514,7 +513,10 @@ public: * criterium in CConnman::AttemptToEvictConnection. */ std::atomic<std::chrono::microseconds> m_min_ping_time{std::chrono::microseconds::max()}; - CNode(NodeId id, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion); + CNode(NodeId id, std::shared_ptr<Sock> sock, const CAddress& addrIn, + uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, + const CAddress& addrBindIn, const std::string& addrNameIn, + ConnectionType conn_type_in, bool inbound_onion); CNode(const CNode&) = delete; CNode& operator=(const CNode&) = delete; @@ -572,11 +574,6 @@ public: void CopyStats(CNodeStats& stats) EXCLUSIVE_LOCKS_REQUIRED(!m_subver_mutex, !m_addr_local_mutex, !cs_vSend, !cs_vRecv); - ServiceFlags GetLocalServices() const - { - return nLocalServices; - } - std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ @@ -591,23 +588,6 @@ private: const ConnectionType m_conn_type; std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; - //! Services offered to this peer. - //! - //! This is supplied by the parent CConnman during peer connection - //! (CConnman::ConnectNode()) from its attribute of the same name. - //! - //! This is const because there is no protocol defined for renegotiating - //! services initially offered to a peer. The set of local services we - //! offer should not change after initialization. - //! - //! An interesting example of this is NODE_NETWORK and initial block - //! download: a node which starts up from scratch doesn't have any blocks - //! to serve, but still advertises NODE_NETWORK because it will eventually - //! fulfill this role after IBD completes. P2P code is written in such a - //! way that it can gracefully handle peers who don't make good on their - //! service advertisements. - const ServiceFlags nLocalServices; - std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread // Our address, as reported by the peer @@ -625,7 +605,7 @@ class NetEventsInterface { public: /** Initialize a peer (setup state, queue any initial messages) */ - virtual void InitializeNode(CNode* pnode) = 0; + virtual void InitializeNode(CNode& node, ServiceFlags our_services) = 0; /** Handle removal of a peer (clear state) */ virtual void FinalizeNode(const CNode& node) = 0; @@ -1035,16 +1015,14 @@ private: std::map<uint64_t, CachedAddrResponse> m_addr_response_caches; /** - * Services this instance offers. + * Services this node offers. * - * This data is replicated in each CNode instance we create during peer - * connection (in ConnectNode()) under a member also called - * nLocalServices. + * This data is replicated in each Peer instance we create. * * This data is not marked const, but after being set it should not - * change. See the note in CNode::nLocalServices documentation. + * change. * - * \sa CNode::nLocalServices + * \sa Peer::our_services */ ServiceFlags nLocalServices; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c33dd29923..0e10fa5f9d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -29,6 +29,7 @@ #include <scheduler.h> #include <streams.h> #include <sync.h> +#include <timedata.h> #include <tinyformat.h> #include <txmempool.h> #include <txorphanage.h> @@ -207,6 +208,23 @@ struct Peer { /** Same id as the CNode object for this peer */ const NodeId m_id{0}; + /** Services we offered to this peer. + * + * This is supplied by CConnman during peer initialization. It's const + * because there is no protocol defined for renegotiating services + * initially offered to a peer. The set of local services we offer should + * not change after initialization. + * + * An interesting example of this is NODE_NETWORK and initial block + * download: a node which starts up from scratch doesn't have any blocks + * to serve, but still advertises NODE_NETWORK because it will eventually + * fulfill this role after IBD completes. P2P code is written in such a + * way that it can gracefully handle peers who don't make good on their + * service advertisements. */ + const ServiceFlags m_our_services; + /** Services this peer offered to us. */ + std::atomic<ServiceFlags> m_their_services{NODE_NONE}; + /** Protects misbehavior data members */ Mutex m_misbehavior_mutex; /** Accumulated misbehavior score for this peer */ @@ -358,10 +376,11 @@ struct Peer { std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); /** Time of the last getheaders message to this peer */ - std::atomic<NodeClock::time_point> m_last_getheaders_timestamp{NodeSeconds{}}; + NodeClock::time_point m_last_getheaders_timestamp{}; - Peer(NodeId id) + explicit Peer(NodeId id, ServiceFlags our_services) : m_id{id} + , m_our_services{our_services} {} private: @@ -410,8 +429,6 @@ struct CNodeState { bool m_requested_hb_cmpctblocks{false}; /** Whether this peer will send us cmpctblocks if we request them. */ bool m_provides_cmpctblocks{false}; - //! Whether this peer can give us witnesses - bool fHaveWitness{false}; /** State used to enforce CHAIN_SYNC_TIMEOUT and EXTRA_PEER_CHECK_INTERVAL logic. * @@ -482,7 +499,7 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex); /** Implement NetEventsInterface */ - void InitializeNode(CNode* pnode) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + void InitializeNode(CNode& node, ServiceFlags our_services) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void FinalizeNode(const CNode& node) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex); @@ -492,7 +509,8 @@ public: /** Implement PeerManager */ void StartScheduledTasks(CScheduler& scheduler) override; void CheckForStaleTipAndEvictPeers() override; - std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override; + std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -578,7 +596,7 @@ private: */ bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer); /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ - void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); + void HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast); /** Update peer state based on received headers message */ void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); @@ -657,7 +675,7 @@ private: /** Get a pointer to a mutable CNodeState. */ CNodeState* State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - uint32_t GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + uint32_t GetFetchFlags(const Peer& peer) const; std::atomic<std::chrono::microseconds> m_next_inv_to_inbounds{0us}; @@ -778,7 +796,7 @@ private: /** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has * at most count entries. */ - void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); @@ -848,6 +866,7 @@ private: * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] filter_type The filter type the request is for. Must be basic filters. * @param[in] start_height The start height for the request @@ -857,7 +876,7 @@ private: * @param[out] filter_index The filter index, if the request can be serviced. * @return True if the request can be serviced. */ - bool PrepareBlockFilterRequest(CNode& peer, + bool PrepareBlockFilterRequest(CNode& node, Peer& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, @@ -868,30 +887,33 @@ private: * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFilters(CNode& peer, CDataStream& vRecv); + void ProcessGetCFilters(CNode& node, Peer& peer, CDataStream& vRecv); /** * Handle a cfheaders request. * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv); + void ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv); /** * Handle a getcfcheckpt request. * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv); + void ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv); /** Checks if address relay is permitted with peer. If needed, initializes * the m_addr_known bloom filter and sets m_addr_relay_enabled to true. @@ -955,6 +977,26 @@ static void AddKnownTx(Peer& peer, const uint256& hash) tx_relay->m_tx_inventory_known_filter.insert(hash); } +/** Whether this peer can serve us blocks. */ +static bool CanServeBlocks(const Peer& peer) +{ + return peer.m_their_services & (NODE_NETWORK|NODE_NETWORK_LIMITED); +} + +/** Whether this peer can only serve limited recent blocks (e.g. because + * it prunes old blocks) */ +static bool IsLimitedPeer(const Peer& peer) +{ + return (!(peer.m_their_services & NODE_NETWORK) && + (peer.m_their_services & NODE_NETWORK_LIMITED)); +} + +/** Whether this peer can serve us witness data */ +static bool CanServeWitnesses(const Peer& peer) +{ + return peer.m_their_services & NODE_WITNESS; +} + std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::microseconds now, std::chrono::seconds average_interval) { @@ -1148,17 +1190,17 @@ void PeerManagerImpl::UpdateBlockAvailability(NodeId nodeid, const uint256 &hash } } -void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) +void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) { if (count == 0) return; vBlocks.reserve(vBlocks.size() + count); - CNodeState *state = State(nodeid); + CNodeState *state = State(peer.m_id); assert(state != nullptr); // Make sure pindexBestKnownBlock is up to date, we'll need it. - ProcessBlockAvailability(nodeid); + ProcessBlockAvailability(peer.m_id); if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has nothing interesting. @@ -1206,7 +1248,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count // We consider the chain that this peer is on invalid. return; } - if (!State(nodeid)->fHaveWitness && DeploymentActiveAt(*pindex, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { + if (!CanServeWitnesses(peer) && DeploymentActiveAt(*pindex, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { // We wouldn't download this block or its descendants from this peer. return; } @@ -1217,7 +1259,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count // The block is not already downloaded, and not yet in flight. if (pindex->nHeight > nWindowEnd) { // We reached the end of the window. - if (vBlocks.size() == 0 && waitingfor != nodeid) { + if (vBlocks.size() == 0 && waitingfor != peer.m_id) { // We aren't able to fetch anything, but we would be if the download window was one larger. nodeStaller = waitingfor; } @@ -1239,10 +1281,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) { - // Note that pnode->GetLocalServices() is a reflection of the local - // services we were offering when the CNode object was created for this - // peer. - uint64_t my_services{pnode.GetLocalServices()}; + uint64_t my_services{peer.m_our_services}; const int64_t nTime{count_seconds(GetTime<std::chrono::seconds>())}; uint64_t nonce = pnode.GetLocalNonce(); const int nNodeStartingHeight{m_best_height}; @@ -1299,21 +1338,21 @@ void PeerManagerImpl::UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_s if (state) state->m_last_block_announcement = time_in_seconds; } -void PeerManagerImpl::InitializeNode(CNode *pnode) +void PeerManagerImpl::InitializeNode(CNode& node, ServiceFlags our_services) { - NodeId nodeid = pnode->GetId(); + NodeId nodeid = node.GetId(); { LOCK(cs_main); - m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(pnode->IsInboundConn())); + m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(node.IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } - PeerRef peer = std::make_shared<Peer>(nodeid); + PeerRef peer = std::make_shared<Peer>(nodeid, our_services); { LOCK(m_peer_mutex); m_peer_map.emplace_hint(m_peer_map.end(), nodeid, peer); } - if (!pnode->IsInboundConn()) { - PushNodeVersion(*pnode, *peer); + if (!node.IsInboundConn()) { + PushNodeVersion(node, *peer); } } @@ -1431,6 +1470,7 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; + stats.their_services = peer->m_their_services; stats.m_starting_height = peer->m_starting_height; // It is common for nodes with good ping times to suddenly become lagged, // due to a new block arriving or other large transfer. @@ -1585,12 +1625,14 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl if (fImporting) return "Importing..."; if (fReindex) return "Reindexing..."; - LOCK(cs_main); // Ensure this peer exists and hasn't been disconnected - CNodeState* state = State(peer_id); - if (state == nullptr) return "Peer does not exist"; + PeerRef peer = GetPeerRef(peer_id); + if (peer == nullptr) return "Peer does not exist"; + // Ignore pre-segwit peers - if (!state->fHaveWitness) return "Pre-SegWit peer"; + if (!CanServeWitnesses(*peer)) return "Pre-SegWit peer"; + + LOCK(cs_main); // Mark block as in-flight unless it already is (for this peer). // If a block was already in-flight for a different peer, its BLOCKTXN @@ -1974,7 +2016,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold if (!pfrom.HasPermission(NetPermissionFlags::NoBan) && ( - (((pfrom.GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom.GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (m_chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) + (((peer.m_our_services & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((peer.m_our_services & NODE_NETWORK) != NODE_NETWORK) && (m_chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) )) { LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold, disconnect peer=%d\n", pfrom.GetId()); //disconnect node and prevent it from stalling (would otherwise wait for the missing block) @@ -2191,10 +2233,10 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic } } -uint32_t PeerManagerImpl::GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main) +uint32_t PeerManagerImpl::GetFetchFlags(const Peer& peer) const { uint32_t nFetchFlags = 0; - if (State(pfrom.GetId())->fHaveWitness) { + if (CanServeWitnesses(peer)) { nFetchFlags |= MSG_WITNESS_FLAG; } return nFetchFlags; @@ -2276,7 +2318,7 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc // Only allow a new getheaders message to go out if we don't have a recent // one already in-flight - if (current_time - peer.m_last_getheaders_timestamp.load() > HEADERS_RESPONSE_TIME) { + if (current_time - peer.m_last_getheaders_timestamp > HEADERS_RESPONSE_TIME) { m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); peer.m_last_getheaders_timestamp = current_time; return true; @@ -2289,7 +2331,7 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc * We require that the given tip have at least as much work as our tip, and for * our current tip to be "close to synced" (see CanDirectFetch()). */ -void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast) +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); @@ -2304,7 +2346,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { + (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || CanServeWitnesses(peer))) { // We don't have this block, and it's not yet in flight. vToFetch.push_back(pindexWalk); } @@ -2326,7 +2368,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* // Can't download any more from this peer break; } - uint32_t nFetchFlags = GetFetchFlags(pfrom); + uint32_t nFetchFlags = GetFetchFlags(peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); BlockRequested(pfrom.GetId(), *pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", @@ -2471,7 +2513,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); // Consider immediately downloading blocks. - HeadersDirectFetchBlocks(pfrom, pindexLast); + HeadersDirectFetchBlocks(pfrom, peer, pindexLast); return; } @@ -2555,7 +2597,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) } } -bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, +bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, @@ -2563,11 +2605,11 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, { const bool supported_filter_type = (filter_type == BlockFilterType::BASIC && - (peer.GetLocalServices() & NODE_COMPACT_FILTERS)); + (peer.m_our_services & NODE_COMPACT_FILTERS)); if (!supported_filter_type) { LogPrint(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", - peer.GetId(), static_cast<uint8_t>(filter_type)); - peer.fDisconnect = true; + node.GetId(), static_cast<uint8_t>(filter_type)); + node.fDisconnect = true; return false; } @@ -2578,8 +2620,8 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index)) { LogPrint(BCLog::NET, "peer %d requested invalid block hash: %s\n", - peer.GetId(), stop_hash.ToString()); - peer.fDisconnect = true; + node.GetId(), stop_hash.ToString()); + node.fDisconnect = true; return false; } } @@ -2588,14 +2630,14 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, if (start_height > stop_height) { LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ "start height %d and stop height %d\n", - peer.GetId(), start_height, stop_height); - peer.fDisconnect = true; + node.GetId(), start_height, stop_height); + node.fDisconnect = true; return false; } if (stop_height - start_height >= max_height_diff) { LogPrint(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", - peer.GetId(), stop_height - start_height + 1, max_height_diff); - peer.fDisconnect = true; + node.GetId(), stop_height - start_height + 1, max_height_diff); + node.fDisconnect = true; return false; } @@ -2608,7 +2650,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, return true; } -void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFilters(CNode& node,Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -2620,7 +2662,7 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, start_height, stop_hash, MAX_GETCFILTERS_SIZE, stop_index, filter_index)) { return; } @@ -2633,13 +2675,13 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) } for (const auto& filter : filters) { - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFILTER, filter); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } } -void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -2651,7 +2693,7 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, start_height, stop_hash, MAX_GETCFHEADERS_SIZE, stop_index, filter_index)) { return; } @@ -2674,16 +2716,16 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) return; } - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFHEADERS, filter_type_ser, stop_index->GetBlockHash(), prev_header, filter_hashes); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } -void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint256 stop_hash; @@ -2694,7 +2736,7 @@ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, /*start_height=*/0, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, /*start_height=*/0, stop_hash, /*max_height_diff=*/std::numeric_limits<uint32_t>::max(), stop_index, filter_index)) { return; @@ -2715,12 +2757,12 @@ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) } } - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFCHECKPT, filter_type_ser, stop_index->GetBlockHash(), headers); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing) @@ -2842,7 +2884,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - pfrom.nServices = nServices; + pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); + peer->m_their_services = nServices; pfrom.SetAddrLocal(addrMe); { LOCK(pfrom.m_subver_mutex); @@ -2850,18 +2893,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } peer->m_starting_height = starting_height; - // set nodes not relaying blocks and tx and not serving (parts) of the historical blockchain as "clients" - pfrom.fClient = (!(nServices & NODE_NETWORK) && !(nServices & NODE_NETWORK_LIMITED)); - - // set nodes not capable of serving the complete blockchain history as "limited nodes" - pfrom.m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); - // We only initialize the m_tx_relay data structure if: // - this isn't an outbound block-relay-only connection; and // - fRelay=true or we're offering NODE_BLOOM to this peer // (NODE_BLOOM means that the peer may turn on tx relay later) if (!pfrom.IsBlockOnlyConn() && - (fRelay || (pfrom.GetLocalServices() & NODE_BLOOM))) { + (fRelay || (peer->m_our_services & NODE_BLOOM))) { auto* const tx_relay = peer->SetTxRelay(); { LOCK(tx_relay->m_bloom_filter_mutex); @@ -2870,17 +2907,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fRelay) pfrom.m_relays_txs = true; } - if((nServices & NODE_WITNESS)) - { - LOCK(cs_main); - State(pfrom.GetId())->fHaveWitness = true; - } - // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); CNodeState* state = State(pfrom.GetId()); - state->fPreferredDownload = (!pfrom.IsInboundConn() || pfrom.HasPermission(NetPermissionFlags::NoBan)) && !pfrom.IsAddrFetchConn() && !pfrom.fClient; + state->fPreferredDownload = (!pfrom.IsInboundConn() || pfrom.HasPermission(NetPermissionFlags::NoBan)) && !pfrom.IsAddrFetchConn() && CanServeBlocks(*peer); m_num_preferred_download_peers += state->fPreferredDownload; } @@ -2899,7 +2930,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // indicate to the peer that we will participate in addr relay. if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { - CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); + CAddress addr{GetLocalAddress(pfrom.addr), peer->m_our_services, AdjustedTime()}; FastRandomContext insecure_rand; if (addr.IsRoutable()) { @@ -3104,8 +3135,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Store the new addresses std::vector<CAddress> vAddrOk; - int64_t nNow = GetAdjustedTime(); - int64_t nSince = nNow - 10 * 60; + const auto current_a_time{AdjustedTime()}; // Update/increment addr rate limiting bucket. const auto current_time{GetTime<std::chrono::microseconds>()}; @@ -3141,8 +3171,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!MayHaveUsefulAddressDB(addr.nServices) && !HasAllDesirableServiceFlags(addr.nServices)) continue; - if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) - addr.nTime = nNow - 5 * 24 * 60 * 60; + if (addr.nTime <= NodeSeconds{100000000s} || addr.nTime > current_a_time + 10min) { + addr.nTime = current_a_time - 5 * 24h; + } AddAddressKnown(*peer, addr); if (m_banman && (m_banman->IsDiscouraged(addr) || m_banman->IsBanned(addr))) { // Do not process banned/discouraged addresses beyond remembering we received them @@ -3150,7 +3181,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } ++num_proc; bool fReachable = IsReachable(addr); - if (addr.nTime > nSince && !peer->m_getaddr_sent && vAddr.size() <= 10 && addr.IsRoutable()) { + if (addr.nTime > current_a_time - 10min && !peer->m_getaddr_sent && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes RelayAddress(pfrom.GetId(), addr, fReachable); } @@ -3163,7 +3194,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "Received addr: %u addresses (%u processed, %u rate-limited) from peer=%d\n", vAddr.size(), num_proc, num_rate_limit, pfrom.GetId()); - m_addrman.Add(vAddrOk, pfrom.addr, 2 * 60 * 60); + m_addrman.Add(vAddrOk, pfrom.addr, 2h); if (vAddr.size() < 1000) peer->m_getaddr_sent = false; // AddrFetch: Require multiple addresses to avoid disconnecting on self-announcements @@ -3600,10 +3631,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetIntArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = m_orphanage.LimitOrphans(nMaxOrphanTx); - if (nEvicted > 0) { - LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); - } + m_orphanage.LimitOrphans(nMaxOrphanTx); } else { LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); // We will continue to reject this tx since it has rejected @@ -3758,7 +3786,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return; @@ -3794,7 +3822,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } @@ -3837,7 +3865,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } else { @@ -3921,7 +3949,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -3974,7 +4002,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Assume that this is in response to any outstanding getheaders // request we may have sent, and clear out the time of our last request - peer->m_last_getheaders_timestamp.store(NodeSeconds{}); + peer->m_last_getheaders_timestamp = {}; std::vector<CBlockHeader> headers; @@ -4061,7 +4089,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::MEMPOOL) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) + if (!(peer->m_our_services & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { @@ -4164,7 +4192,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERLOAD) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4189,7 +4217,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERADD) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4217,7 +4245,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERCLEAR) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4248,17 +4276,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv); + ProcessGetCFilters(pfrom, *peer, vRecv); return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv); + ProcessGetCFHeaders(pfrom, *peer, vRecv); return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv); + ProcessGetCFCheckPt(pfrom, *peer, vRecv); return; } @@ -4654,9 +4682,10 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros if (peer.m_next_local_addr_send != 0us) { peer.m_addr_known->reset(); } - if (std::optional<CAddress> local_addr = GetLocalAddrForPeer(&node)) { + if (std::optional<CService> local_service = GetLocalAddrForPeer(node)) { + CAddress local_addr{*local_service, peer.m_our_services, AdjustedTime()}; FastRandomContext insecure_rand; - PushAddress(peer, *local_addr, insecure_rand); + PushAddress(peer, local_addr, insecure_rand); } peer.m_next_local_addr_send = GetExponentialRand(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } @@ -4840,7 +4869,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) bool sync_blocks_and_headers_from_peer = false; if (state.fPreferredDownload) { sync_blocks_and_headers_from_peer = true; - } else if (!pto->fClient && !pto->IsAddrFetchConn()) { + } else if (CanServeBlocks(*peer) && !pto->IsAddrFetchConn()) { // Typically this is an inbound peer. If we don't have any outbound // peers, or if we aren't downloading any blocks from such peers, // then allow block downloads from this peer, too. @@ -4855,7 +4884,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } - if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { + if (!state.fSyncStarted && CanServeBlocks(*peer) && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { const CBlockIndex* pindexStart = m_chainman.m_best_header; @@ -5232,12 +5261,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector<CInv> vGetData; - if (!pto->fClient && ((sync_blocks_and_headers_from_peer && !pto->m_limited_node) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(*pto); + uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); BlockRequested(pto->GetId(), *pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), @@ -5264,7 +5293,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!AlreadyHaveTx(gtxid)) { LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", gtxid.GetHash().ToString(), pto->GetId()); - vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), gtxid.GetHash()); + vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); if (vGetData.size() >= MAX_GETDATA_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); vGetData.clear(); diff --git a/src/net_processing.h b/src/net_processing.h index 5fbae98c27..bcda9614d4 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -34,6 +34,7 @@ struct CNodeStateStats { uint64_t m_addr_processed = 0; uint64_t m_addr_rate_limited = 0; bool m_addr_relay_enabled{false}; + ServiceFlags their_services; }; class PeerManager : public CValidationInterface, public NetEventsInterface diff --git a/src/netaddress.h b/src/netaddress.h index 47ba045334..e52beb783d 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <compat.h> +#include <compat/compat.h> #include <crypto/siphash.h> #include <prevector.h> #include <random.h> diff --git a/src/netbase.cpp b/src/netbase.cpp index 030f462ed9..4b8d2f8d0c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -5,7 +5,7 @@ #include <netbase.h> -#include <compat.h> +#include <compat/compat.h> #include <sync.h> #include <tinyformat.h> #include <util/sock.h> diff --git a/src/netbase.h b/src/netbase.h index bf7522210d..fadc8b418e 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <compat.h> +#include <compat/compat.h> #include <netaddress.h> #include <serialize.h> #include <util/sock.h> diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index cadafcaa8d..601d0bdf58 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -21,6 +21,7 @@ #include <util/system.h> #include <validation.h> +#include <map> #include <unordered_map> namespace node { @@ -471,7 +472,7 @@ static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const fileout << blockundo; // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + HashWriter hasher{}; hasher << hashBlock; hasher << blockundo; fileout << hasher.GetHash(); @@ -823,7 +824,7 @@ struct CImportingNow { } }; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -834,6 +835,9 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile // -reindex if (fReindex) { int nFile = 0; + // Map of disk positions for blocks with unknown parent (only used for reindex); + // parent hash -> child disk position, multiple children can have the same parent. + std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; while (true) { FlatFilePos pos(nFile, 0); if (!fs::exists(GetBlockPosFilename(pos))) { @@ -844,7 +848,7 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile break; // This error is logged in OpenBlockFile } LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos); + chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -893,6 +897,6 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile return; } } // End scope of CImportingNow - chainman.ActiveChainstate().LoadMempool(args); + chainman.ActiveChainstate().LoadMempool(mempool_path); } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index e017f3f427..9b76371aae 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -211,7 +211,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args); +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 54ba5b7966..ad9293f172 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -4,66 +4,75 @@ #include <node/chainstate.h> +#include <chain.h> +#include <coins.h> #include <consensus/params.h> #include <node/blockstorage.h> +#include <node/caches.h> +#include <sync.h> +#include <threadsafety.h> +#include <tinyformat.h> +#include <txdb.h> +#include <uint256.h> +#include <util/time.h> +#include <util/translation.h> #include <validation.h> +#include <algorithm> +#include <atomic> +#include <cassert> +#include <memory> +#include <vector> + namespace node { -std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, - ChainstateManager& chainman, - CTxMemPool* mempool, - bool fPruneMode, - bool fReindexChainState, - int64_t nBlockTreeDBCache, - int64_t nCoinDBCache, - int64_t nCoinCacheUsage, - bool block_tree_db_in_memory, - bool coins_db_in_memory, - std::function<bool()> shutdown_requested, - std::function<void()> coins_error_cb) +ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); - chainman.InitializeChainstate(mempool); - chainman.m_total_coinstip_cache = nCoinCacheUsage; - chainman.m_total_coinsdb_cache = nCoinDBCache; + chainman.InitializeChainstate(options.mempool); + chainman.m_total_coinstip_cache = cache_sizes.coins; + chainman.m_total_coinsdb_cache = cache_sizes.coins_db; auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); - pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, block_tree_db_in_memory, fReset)); + pblocktree.reset(new CBlockTreeDB(cache_sizes.block_tree_db, options.block_tree_db_in_memory, options.reindex)); - if (fReset) { + if (options.reindex) { pblocktree->WriteReindexing(true); //If we're reindexing in prune mode, wipe away unusable block files and all undo data files - if (fPruneMode) + if (options.prune) { CleanupBlockRevFiles(); + } } - if (shutdown_requested && shutdown_requested()) return ChainstateLoadingError::SHUTDOWN_PROBED; + if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets fReindex based on the disk flag! - // From here on out fReindex and fReset mean something different! + // Note that it also sets fReindex global based on the disk flag! + // From here on, fReindex and options.reindex values may be different! if (!chainman.LoadBlockIndex()) { - if (shutdown_requested && shutdown_requested()) return ChainstateLoadingError::SHUTDOWN_PROBED; - return ChainstateLoadingError::ERROR_LOADING_BLOCK_DB; + if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; + return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; } if (!chainman.BlockIndex().empty() && !chainman.m_blockman.LookupBlockIndex(chainman.GetConsensus().hashGenesisBlock)) { - return ChainstateLoadingError::ERROR_BAD_GENESIS_BLOCK; + // If the loaded chain has a wrong genesis, bail out immediately + // (we're likely using a testnet datadir, or the other way around). + return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Incorrect or no genesis block found. Wrong datadir for network?")}; } // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. - if (chainman.m_blockman.m_have_pruned && !fPruneMode) { - return ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX; + if (chainman.m_blockman.m_have_pruned && !options.prune) { + return {ChainstateLoadStatus::FAILURE, _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain")}; } // At this point blocktree args are consistent with what's on disk. @@ -71,7 +80,7 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, // (otherwise we use the one already on disk). // This is called again in ThreadImport after the reindex completes. if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { - return ChainstateLoadingError::ERROR_LOAD_GENESIS_BLOCK_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } // At this point we're either in reindex or we've loaded a useful @@ -79,57 +88,56 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, for (CChainState* chainstate : chainman.GetAll()) { chainstate->InitCoinsDB( - /*cache_size_bytes=*/nCoinDBCache, - /*in_memory=*/coins_db_in_memory, - /*should_wipe=*/fReset || fReindexChainState); + /*cache_size_bytes=*/cache_sizes.coins_db, + /*in_memory=*/options.coins_db_in_memory, + /*should_wipe=*/options.reindex || options.reindex_chainstate); - if (coins_error_cb) { - chainstate->CoinsErrorCatcher().AddReadErrCallback(coins_error_cb); + if (options.coins_error_cb) { + chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); } // Refuse to load unsupported database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (chainstate->CoinsDB().NeedsUpgrade()) { - return ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED; + return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Unsupported chainstate database format found. " + "Please restart with -reindex-chainstate. This will " + "rebuild the chainstate database.")}; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->ReplayBlocks()) { - return ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.")}; } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(nCoinCacheUsage); + chainstate->InitCoinsCache(cache_sizes.coins); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s best block if (!chainstate->LoadChainTip()) { - return ChainstateLoadingError::ERROR_LOADCHAINTIP_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } assert(chainstate->m_chain.Tip() != nullptr); } } - if (!fReset) { + if (!options.reindex) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { - return ChainstateLoadingError::ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED; - } + return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), + chainman.GetConsensus().SegwitHeight)}; + }; } - return std::nullopt; + return {ChainstateLoadStatus::SUCCESS, {}}; } -std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManager& chainman, - bool fReset, - bool fReindexChainState, - int check_blocks, - int check_level) +ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); @@ -138,18 +146,20 @@ std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManage if (!is_coinsview_empty(chainstate)) { const CBlockIndex* tip = chainstate->m_chain.Tip(); if (tip && tip->nTime > GetTime() + MAX_FUTURE_BLOCK_TIME) { - return ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE; + return {ChainstateLoadStatus::FAILURE, _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct")}; } if (!CVerifyDB().VerifyDB( *chainstate, chainman.GetConsensus(), chainstate->CoinsDB(), - check_level, - check_blocks)) { - return ChainstateLoadVerifyError::ERROR_CORRUPTED_BLOCK_DB; + options.check_level, + options.check_blocks)) { + return {ChainstateLoadStatus::FAILURE, _("Corrupted block database detected")}; } } } - return std::nullopt; + return {ChainstateLoadStatus::SUCCESS, {}}; } } // namespace node diff --git a/src/node/chainstate.h b/src/node/chainstate.h index ff7935e8e0..2289310ece 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -5,30 +5,41 @@ #ifndef BITCOIN_NODE_CHAINSTATE_H #define BITCOIN_NODE_CHAINSTATE_H +#include <util/translation.h> +#include <validation.h> + #include <cstdint> #include <functional> -#include <optional> +#include <tuple> -class ChainstateManager; class CTxMemPool; -namespace Consensus { -struct Params; -} // namespace Consensus namespace node { -enum class ChainstateLoadingError { - ERROR_LOADING_BLOCK_DB, - ERROR_BAD_GENESIS_BLOCK, - ERROR_PRUNED_NEEDS_REINDEX, - ERROR_LOAD_GENESIS_BLOCK_FAILED, - ERROR_CHAINSTATE_UPGRADE_FAILED, - ERROR_REPLAYBLOCKS_FAILED, - ERROR_LOADCHAINTIP_FAILED, - ERROR_GENERIC_BLOCKDB_OPEN_FAILED, - ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED, - SHUTDOWN_PROBED, + +struct CacheSizes; + +struct ChainstateLoadOptions { + CTxMemPool* mempool{nullptr}; + bool block_tree_db_in_memory{false}; + bool coins_db_in_memory{false}; + bool reindex{false}; + bool reindex_chainstate{false}; + bool prune{false}; + int64_t check_blocks{DEFAULT_CHECKBLOCKS}; + int64_t check_level{DEFAULT_CHECKLEVEL}; + std::function<bool()> check_interrupt; + std::function<void()> coins_error_cb; }; +//! Chainstate load status. Simple applications can just check for the success +//! case, and treat other cases as errors. More complex applications may want to +//! try reindexing in the generic failure case, and pass an interrupt callback +//! and exit cleanly in the interrupted case. +enum class ChainstateLoadStatus { SUCCESS, FAILURE, FAILURE_INCOMPATIBLE_DB, INTERRUPTED }; + +//! Chainstate load status code and optional error string. +using ChainstateLoadResult = std::tuple<ChainstateLoadStatus, bilingual_str>; + /** This sequence can have 4 types of outcomes: * * 1. Success @@ -40,45 +51,11 @@ enum class ChainstateLoadingError { * 4. Hard failure * - a failure that definitively cannot be recovered from with a reindex * - * Currently, LoadChainstate returns a std::optional<ChainstateLoadingError> - * which: - * - * - if has_value() - * - Either "Soft failure", "Hard failure", or "Shutdown requested", - * differentiable by the specific enumerator. - * - * Note that a return value of SHUTDOWN_PROBED means ONLY that "during - * this sequence, when we explicitly checked shutdown_requested() at - * arbitrary points, one of those calls returned true". Therefore, a - * return value other than SHUTDOWN_PROBED does not guarantee that - * shutdown hasn't been called indirectly. - * - else - * - Success! + * LoadChainstate returns a (status code, error string) tuple. */ -std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, - ChainstateManager& chainman, - CTxMemPool* mempool, - bool fPruneMode, - bool fReindexChainState, - int64_t nBlockTreeDBCache, - int64_t nCoinDBCache, - int64_t nCoinCacheUsage, - bool block_tree_db_in_memory, - bool coins_db_in_memory, - std::function<bool()> shutdown_requested = nullptr, - std::function<void()> coins_error_cb = nullptr); - -enum class ChainstateLoadVerifyError { - ERROR_BLOCK_FROM_FUTURE, - ERROR_CORRUPTED_BLOCK_DB, - ERROR_GENERIC_FAILURE, -}; - -std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManager& chainman, - bool fReset, - bool fReindexChainState, - int check_blocks, - int check_level); +ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options); +ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options); } // namespace node #endif // BITCOIN_NODE_CHAINSTATE_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 93aa9e526b..46d45377fa 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -19,6 +19,7 @@ #include <netaddress.h> #include <netbase.h> #include <node/blockstorage.h> +#include <kernel/chain.h> #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> @@ -35,7 +36,6 @@ #include <shutdown.h> #include <support/allocators/secure.h> #include <sync.h> -#include <timedata.h> #include <txmempool.h> #include <uint256.h> #include <univalue.h> @@ -401,6 +401,7 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; + if (block.m_locator) { *block.m_locator = active.GetLocator(index); } if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); if (block.m_data) { REVERSE_LOCK(lock); @@ -426,11 +427,11 @@ public: } void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->blockConnected(*block, index->nHeight); + m_notifications->blockConnected(kernel::MakeBlockInfo(index, block.get())); } void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->blockDisconnected(*block, index->nHeight); + m_notifications->blockDisconnected(kernel::MakeBlockInfo(index, block.get())); } void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { @@ -507,7 +508,7 @@ public: std::optional<int> getHeight() override { LOCK(::cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); int height = active.Height(); if (height >= 0) { return height; @@ -517,7 +518,7 @@ public: uint256 getBlockHash(int height) override { LOCK(::cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); CBlockIndex* block = active[height]; assert(block); return block->GetBlockHash(); @@ -525,14 +526,14 @@ public: bool haveBlockOnDisk(int height) override { LOCK(::cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); CBlockIndex* block = active[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } CBlockLocator getTipLocator() override { LOCK(::cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); return active.GetLocator(); } CBlockLocator getActiveChainLocator(const uint256& block_hash) override @@ -545,7 +546,7 @@ public: std::optional<int> findLocatorFork(const CBlockLocator& locator) override { LOCK(::cs_main); - const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); + const CChainState& active = chainman().ActiveChainstate(); if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; } @@ -554,20 +555,20 @@ public: bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - return FillBlock(m_node.chainman->m_blockman.LookupBlockIndex(hash), block, lock, active); + const CChain& active = chainman().ActiveChain(); + return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, active); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active); } bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override { WAIT_LOCK(cs_main, lock); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (const CBlockIndex* block = m_node.chainman->m_blockman.LookupBlockIndex(block_hash)) { + const CChain& active = chainman().ActiveChain(); + if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { return FillBlock(ancestor, ancestor_out, lock, active); } @@ -577,18 +578,18 @@ public: bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override { WAIT_LOCK(cs_main, lock); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block = m_node.chainman->m_blockman.LookupBlockIndex(block_hash); - const CBlockIndex* ancestor = m_node.chainman->m_blockman.LookupBlockIndex(ancestor_hash); + const CChain& active = chainman().ActiveChain(); + const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = chainman().m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; return FillBlock(ancestor, ancestor_out, lock, active); } bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override { WAIT_LOCK(cs_main, lock); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block1 = m_node.chainman->m_blockman.LookupBlockIndex(block_hash1); - const CBlockIndex* block2 = m_node.chainman->m_blockman.LookupBlockIndex(block_hash2); + const CChain& active = chainman().ActiveChain(); + const CBlockIndex* block1 = chainman().m_blockman.LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = chainman().m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical @@ -701,7 +702,7 @@ public: bool havePruned() override { LOCK(::cs_main); - return m_node.chainman->m_blockman.m_have_pruned; + return chainman().m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } bool isInitialBlockDownload() override { @@ -723,7 +724,7 @@ public: { if (!old_tip.IsNull()) { LOCK(::cs_main); - const CChain& active = Assert(m_node.chainman)->ActiveChain(); + const CChain& active = chainman().ActiveChain(); if (old_tip == active.Tip()->GetBlockHash()) return; } SyncWithValidationInterfaceQueue(); @@ -775,6 +776,12 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } + bool hasAssumedValidChain() override + { + return chainman().IsSnapshotActive(); + } + + NodeContext* context() override { return &m_node; } NodeContext& m_node; }; } // namespace diff --git a/src/node/mempool_persist_args.cpp b/src/node/mempool_persist_args.cpp new file mode 100644 index 0000000000..4e775869c6 --- /dev/null +++ b/src/node/mempool_persist_args.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/mempool_persist_args.h> + +#include <fs.h> +#include <util/system.h> +#include <validation.h> + +namespace node { + +bool ShouldPersistMempool(const ArgsManager& argsman) +{ + return argsman.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL); +} + +fs::path MempoolPath(const ArgsManager& argsman) +{ + return argsman.GetDataDirNet() / "mempool.dat"; +} + +} // namespace node diff --git a/src/node/mempool_persist_args.h b/src/node/mempool_persist_args.h new file mode 100644 index 0000000000..f719ec62ab --- /dev/null +++ b/src/node/mempool_persist_args.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H + +#include <fs.h> + +class ArgsManager; + +namespace node { + +/** + * Default for -persistmempool, indicating whether the node should attempt to + * automatically load the mempool on start and save to disk on shutdown + */ +static constexpr bool DEFAULT_PERSIST_MEMPOOL{true}; + +bool ShouldPersistMempool(const ArgsManager& argsman); +fs::path MempoolPath(const ArgsManager& argsman); + +} // namespace node + +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 27a6ab221f..2b940be07e 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -161,13 +161,13 @@ public: unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout) const; + void Write(AutoFile& fileout) const; /** * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); + void Read(AutoFile& filein, int nFileVersion, size_t numBuckets); }; @@ -390,7 +390,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, return median; } -void TxConfirmStats::Write(CAutoFile& fileout) const +void TxConfirmStats::Write(AutoFile& fileout) const { fileout << Using<EncodedDoubleFormatter>(decay); fileout << scale; @@ -400,7 +400,7 @@ void TxConfirmStats::Write(CAutoFile& fileout) const fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg); } -void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) +void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets) { // Read data file and do some very basic sanity checking // buckets and bucketMap are not updated yet, so don't access them @@ -546,7 +546,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); // If the fee estimation file is present, read recorded estimations - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "rb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")}; if (est_file.IsNull() || !Read(est_file)) { LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } @@ -904,13 +904,13 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation void CBlockPolicyEstimator::Flush() { FlushUnconfirmed(); - CAutoFile est_file(fsbridge::fopen(m_estimation_filepath, "wb"), SER_DISK, CLIENT_VERSION); + AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")}; if (est_file.IsNull() || !Write(est_file)) { LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); } } -bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const +bool CBlockPolicyEstimator::Write(AutoFile& fileout) const { try { LOCK(m_cs_fee_estimator); @@ -935,7 +935,7 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const return true; } -bool CBlockPolicyEstimator::Read(CAutoFile& filein) +bool CBlockPolicyEstimator::Read(AutoFile& filein) { try { LOCK(m_cs_fee_estimator); diff --git a/src/policy/fees.h b/src/policy/fees.h index 9ee5c2938a..e4628bf853 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -20,7 +20,7 @@ #include <string> #include <vector> -class CAutoFile; +class AutoFile; class CTxMemPoolEntry; class TxConfirmStats; @@ -220,11 +220,11 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Write estimation data to a file */ - bool Write(CAutoFile& fileout) const + bool Write(AutoFile& fileout) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Read estimation data from a file */ - bool Read(CAutoFile& filein) + bool Read(AutoFile& filein) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index f7f6ae4480..ec48194ee9 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -7,10 +7,15 @@ #include <consensus/amount.h> #include <hash.h> +#include <script/script.h> +#include <serialize.h> #include <tinyformat.h> +#include <uint256.h> #include <util/strencodings.h> +#include <version.h> -#include <assert.h> +#include <cassert> +#include <stdexcept> std::string COutPoint::ToString() const { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index fb98fb6868..f496ea022e 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -6,13 +6,21 @@ #ifndef BITCOIN_PRIMITIVES_TRANSACTION_H #define BITCOIN_PRIMITIVES_TRANSACTION_H -#include <stdint.h> #include <consensus/amount.h> +#include <prevector.h> #include <script/script.h> #include <serialize.h> #include <uint256.h> +#include <cstddef> +#include <cstdint> +#include <ios> +#include <limits> +#include <memory> +#include <string> #include <tuple> +#include <utility> +#include <vector> /** * A flag that is ORed into the protocol version to designate that a transaction @@ -303,7 +311,7 @@ private: public: /** Convert a CMutableTransaction into a CTransaction. */ explicit CTransaction(const CMutableTransaction& tx); - CTransaction(CMutableTransaction&& tx); + explicit CTransaction(CMutableTransaction&& tx); template <typename Stream> inline void Serialize(Stream& s) const { @@ -368,7 +376,7 @@ struct CMutableTransaction int32_t nVersion; uint32_t nLockTime; - CMutableTransaction(); + explicit CMutableTransaction(); explicit CMutableTransaction(const CTransaction& tx); template <typename Stream> diff --git a/src/protocol.h b/src/protocol.h index da2d24aff3..b85dc0d820 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -11,6 +11,7 @@ #include <serialize.h> #include <streams.h> #include <uint256.h> +#include <util/time.h> #include <cstdint> #include <limits> @@ -352,7 +353,7 @@ static inline bool MayHaveUsefulAddressDB(ServiceFlags services) /** A CService with information about it as peer */ class CAddress : public CService { - static constexpr uint32_t TIME_INIT{100000000}; + static constexpr std::chrono::seconds TIME_INIT{100000000}; /** Historically, CAddress disk serialization stored the CLIENT_VERSION, optionally OR'ed with * the ADDRV2_FORMAT flag to indicate V2 serialization. The first field has since been @@ -382,7 +383,7 @@ class CAddress : public CService public: CAddress() : CService{} {}; CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; - CAddress(CService ipIn, ServiceFlags nServicesIn, uint32_t nTimeIn) : CService{ipIn}, nTime{nTimeIn}, nServices{nServicesIn} {}; + CAddress(CService ipIn, ServiceFlags nServicesIn, NodeSeconds time) : CService{ipIn}, nTime{time}, nServices{nServicesIn} {}; SERIALIZE_METHODS(CAddress, obj) { @@ -415,7 +416,7 @@ public: use_v2 = s.GetVersion() & ADDRV2_FORMAT; } - READWRITE(obj.nTime); + READWRITE(Using<LossyChronoFormatter<uint32_t>>(obj.nTime)); // nServices is serialized as CompactSize in V2; as uint64_t in V1. if (use_v2) { uint64_t services_tmp; @@ -430,8 +431,8 @@ public: SerReadWriteMany(os, ser_action, ReadWriteAsHelper<CService>(obj)); } - //! Always included in serialization. - uint32_t nTime{TIME_INIT}; + //! Always included in serialization. The behavior is unspecified if the value is not representable as uint32_t. + NodeSeconds nTime{TIME_INIT}; //! Serialized as uint64_t in V1, and as CompactSize in V2. ServiceFlags nServices{NODE_NONE}; diff --git a/src/psbt.h b/src/psbt.h index a143a99988..eef7d7dd3b 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -866,6 +866,12 @@ struct PSBTOutput s_tree >> depth; s_tree >> leaf_ver; s_tree >> script; + if (depth > TAPROOT_CONTROL_MAX_NODE_COUNT) { + throw std::ios_base::failure("Output Taproot tree has as leaf greater than Taproot maximum depth"); + } + if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { + throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); + } m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); } if (!m_tap_tree->IsComplete()) { @@ -887,6 +893,9 @@ struct PSBTOutput s >> leaf_hashes; size_t after_hashes = s.size(); size_t hashes_len = before_hashes - after_hashes; + if (hashes_len > value_len) { + throw std::ios_base::failure("Output Taproot BIP32 keypath has an invalid length"); + } size_t origin_len = value_len - hashes_len; m_tap_bip32_paths.emplace(xonly, std::make_pair(leaf_hashes, DeserializeKeyOrigin(s, origin_len))); break; diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 324f681a0a..a4a1be6388 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -211,16 +211,16 @@ bool XOnlyPubKey::VerifySchnorr(const uint256& msg, Span<const unsigned char> si return secp256k1_schnorrsig_verify(secp256k1_context_verify, sigbytes.data(), msg.begin(), 32, &pubkey); } -static const CHashWriter HASHER_TAPTWEAK = TaggedHash("TapTweak"); +static const HashWriter HASHER_TAPTWEAK{TaggedHash("TapTweak")}; uint256 XOnlyPubKey::ComputeTapTweakHash(const uint256* merkle_root) const { if (merkle_root == nullptr) { // We have no scripts. The actual tweak does not matter, but follow BIP341 here to // allow for reproducible tweaking. - return (CHashWriter(HASHER_TAPTWEAK) << m_keydata).GetSHA256(); + return (HashWriter{HASHER_TAPTWEAK} << m_keydata).GetSHA256(); } else { - return (CHashWriter(HASHER_TAPTWEAK) << m_keydata << *merkle_root).GetSHA256(); + return (HashWriter{HASHER_TAPTWEAK} << m_keydata << *merkle_root).GetSHA256(); } } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 27d3a1b9e2..33c60deafb 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -77,8 +77,6 @@ Q_DECLARE_METATYPE(CAmount) Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) -using node::NodeContext; - static void RegisterMetaTypes() { // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 35e32f515b..90f228803c 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -430,8 +430,12 @@ void BitcoinGUI::createActions() if (backup_file.isEmpty()) return; bool wallet_name_ok; - //: Title of the Restore Wallet input dialog (where the wallet name is entered) - QString wallet_name = QInputDialog::getText(this, tr("Restore Name"), tr("Wallet Name:"), QLineEdit::Normal, "", &wallet_name_ok); + /*: Title of pop-up window shown when the user is attempting to ++ restore a wallet. */ + QString title = tr("Restore Wallet"); + //: Label of the input field where the name of the wallet is entered. + QString label = tr("Wallet Name"); + QString wallet_name = QInputDialog::getText(this, title, label, QLineEdit::Normal, "", &wallet_name_ok); if (!wallet_name_ok || wallet_name.isEmpty()) return; auto activity = new RestoreWalletActivity(m_wallet_controller, this); @@ -1350,6 +1354,12 @@ void BitcoinGUI::setEncryptionStatus(int status) { switch(status) { + case WalletModel::NoKeys: + labelWalletEncryptionIcon->hide(); + encryptWalletAction->setChecked(false); + changePassphraseAction->setEnabled(false); + encryptWalletAction->setEnabled(false); + break; case WalletModel::Unencrypted: labelWalletEncryptionIcon->hide(); encryptWalletAction->setChecked(false); diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 38b0ac71a3..e8f39584ad 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -4,7 +4,7 @@ #include <qt/bitcoin.h> -#include <compat.h> +#include <compat/compat.h> #include <util/translation.h> #include <util/url.h> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 42c8b07763..2b6711ca40 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -19,6 +19,7 @@ #include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include <netbase.h> #include <txdb.h> // for -dbcache defaults +#include <util/system.h> #include <chrono> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index b791fd30c4..70fccdef1c 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1162,7 +1162,6 @@ void RPCConsole::updateDetailWidget() if (!stats->nodeStats.addrLocal.empty()) peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); - ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); QString bip152_hb_settings; if (stats->nodeStats.m_bip152_highbandwidth_to) bip152_hb_settings = ts.to; if (stats->nodeStats.m_bip152_highbandwidth_from) bip152_hb_settings += (bip152_hb_settings.isEmpty() ? ts.from : QLatin1Char('/') + ts.from); @@ -1197,6 +1196,7 @@ void RPCConsole::updateDetailWidget() // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { + ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStateStats.their_services)); // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) { ui->peerSyncHeight->setText(QString("%1").arg(stats->nodeStateStats.nSyncHeight)); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index de23f51c92..846fa519ee 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -43,8 +43,6 @@ Q_IMPORT_PLUGIN(QAndroidPlatformIntegrationPlugin) #endif #endif -using node::NodeContext; - const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 11140c5da9..01d84624e8 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -391,9 +391,10 @@ void RestoreWalletActivity::restore(const fs::path& backup_file, const std::stri tr("Restoring Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, backup_file, wallet_name] { - std::unique_ptr<interfaces::Wallet> wallet = node().walletLoader().restoreWallet(backup_file, wallet_name, m_error_message, m_warning_message); + auto wallet{node().walletLoader().restoreWallet(backup_file, wallet_name, m_warning_message)}; - if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); + m_error_message = wallet ? bilingual_str{} : wallet.GetError(); + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(wallet.ReleaseObj()); QTimer::singleShot(0, this, &RestoreWalletActivity::finish); }); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 6b38e207d3..8dc97e66a2 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -213,6 +213,14 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; data.assign(std::istream_iterator<unsigned char>{in}, {}); + + // Some psbt files may be base64 strings in the file rather than binary data + std::string b64_str{data.begin(), data.end()}; + b64_str.erase(b64_str.find_last_not_of(" \t\n\r\f\v") + 1); // Trim trailing whitespace + auto b64_dec = DecodeBase64(b64_str); + if (b64_dec.has_value()) { + data = b64_dec.value(); + } } std::string error; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index bb6079afee..fde136b727 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -306,6 +306,11 @@ WalletModel::EncryptionStatus WalletModel::getEncryptionStatus() const { if(!m_wallet->isCrypted()) { + // A previous bug allowed for watchonly wallets to be encrypted (encryption keys set, but nothing is actually encrypted). + // To avoid misrepresenting the encryption status of such wallets, we only return NoKeys for watchonly wallets that are unencrypted. + if (m_wallet->privateKeysDisabled()) { + return NoKeys; + } return Unencrypted; } else if(m_wallet->isLocked()) diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index a52290dee8..0184fb8ec2 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -71,6 +71,7 @@ public: enum EncryptionStatus { + NoKeys, // wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) Unencrypted, // !wallet->IsCrypted() Locked, // wallet->IsCrypted() && wallet->IsLocked() Unlocked // wallet->IsCrypted() && !wallet->IsLocked() diff --git a/src/random.cpp b/src/random.cpp index fca4b5041a..f92e679a00 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -10,7 +10,7 @@ #include <crypto/sha512.h> #include <support/cleanse.h> #ifdef WIN32 -#include <compat.h> // for Windows API +#include <compat/compat.h> #include <wincrypt.h> #endif #include <logging.h> diff --git a/src/random.h b/src/random.h index b92c29f0be..5fe20c5f76 100644 --- a/src/random.h +++ b/src/random.h @@ -11,6 +11,7 @@ #include <span.h> #include <uint256.h> +#include <cassert> #include <chrono> #include <cstdint> #include <limits> @@ -236,13 +237,19 @@ public: template <typename Tp> Tp rand_uniform_delay(const Tp& time, typename Tp::duration range) { - using Dur = typename Tp::duration; - Dur dur{range.count() > 0 ? /* interval [0..range) */ Dur{randrange(range.count())} : - range.count() < 0 ? /* interval (range..0] */ -Dur{randrange(-range.count())} : - /* interval [0..0] */ Dur{0}}; - return time + dur; + return time + rand_uniform_duration<Tp>(range); } + /** Generate a uniform random duration in the range from 0 (inclusive) to range (exclusive). */ + template <typename Chrono> + typename Chrono::duration rand_uniform_duration(typename Chrono::duration range) noexcept + { + using Dur = typename Chrono::duration; + return range.count() > 0 ? /* interval [0..range) */ Dur{randrange(range.count())} : + range.count() < 0 ? /* interval (range..0] */ -Dur{randrange(-range.count())} : + /* interval [0..0] */ Dur{0}; + }; + // Compatibility with the C++11 UniformRandomBitGenerator concept typedef uint64_t result_type; static constexpr uint64_t min() { return 0; } diff --git a/src/randomenv.cpp b/src/randomenv.cpp index c5dca346d6..9e58180b7a 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -15,7 +15,7 @@ #include <support/cleanse.h> #include <util/time.h> // for GetTime() #ifdef WIN32 -#include <compat.h> // for Windows API +#include <compat/compat.h> #endif #include <algorithm> diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b80cdce3f0..8f116a05ef 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -39,6 +39,7 @@ #include <univalue.h> #include <util/check.h> #include <util/strencodings.h> +#include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -396,7 +397,7 @@ static RPCHelpMan syncwithvalidationinterfacequeue() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { SyncWithValidationInterfaceQueue(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -613,11 +614,12 @@ const RPCResult getblock_vin{ { {RPCResult::Type::BOOL, "generated", "Coinbase or not"}, {RPCResult::Type::NUM, "height", "The height of the prevout"}, - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, @@ -1014,9 +1016,9 @@ static RPCHelpMan gettxout() {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, {RPCResult::Type::STR_AMOUNT, "value", "The transaction value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", ""}, + {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", ""}, + {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, @@ -1054,11 +1056,11 @@ static RPCHelpMan gettxout() LOCK(mempool.cs); CCoinsViewMemPool view(coins_view, mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { - return NullUniValue; + return UniValue::VNULL; } } else { if (!coins_view->GetCoin(out, coin)) { - return NullUniValue; + return UniValue::VNULL; } } @@ -1496,7 +1498,7 @@ static RPCHelpMan preciousblock() throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -1537,7 +1539,7 @@ static RPCHelpMan invalidateblock() throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -1577,7 +1579,7 @@ static RPCHelpMan reconsiderblock() throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -2096,7 +2098,7 @@ static RPCHelpMan scantxoutset() CoinsViewScanReserver reserver; if (reserver.reserve()) { // no scan in progress - return NullUniValue; + return UniValue::VNULL; } result.pushKV("progress", g_scan_progress.load()); return result; @@ -2306,7 +2308,7 @@ static RPCHelpMan dumptxoutset() } FILE* file{fsbridge::fopen(temppath, "wb")}; - CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + AutoFile afile{file}; if (afile.IsNull()) { throw JSONRPCError( RPC_INVALID_PARAMETER, @@ -2327,7 +2329,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot( NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& temppath) { diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 5fbd9d5fd3..a332fd4892 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -55,7 +55,7 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], UniValue CreateUTXOSnapshot( node::NodeContext& node, CChainState& chainstate, - CAutoFile& afile, + AutoFile& afile, const fs::path& path, const fs::path& tmppath); diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index dd1a6441a0..41f386d443 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -6,7 +6,6 @@ #include <core_io.h> #include <policy/feerate.h> #include <policy/fees.h> -#include <policy/policy.h> #include <policy/settings.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -16,7 +15,6 @@ #include <txmempool.h> #include <univalue.h> #include <util/fees.h> -#include <util/system.h> #include <algorithm> #include <array> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 84d43e7818..0ae10b6c39 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,9 +5,12 @@ #include <rpc/blockchain.h> +#include <kernel/mempool_persist.h> + #include <chainparams.h> #include <core_io.h> #include <fs.h> +#include <node/mempool_persist_args.h> #include <policy/rbf.h> #include <policy/settings.h> #include <primitives/transaction.h> @@ -17,8 +20,12 @@ #include <txmempool.h> #include <univalue.h> #include <util/moneystr.h> +#include <util/time.h> + +using kernel::DumpMempool; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::MempoolPath; using node::NodeContext; static RPCHelpMan sendrawtransaction() @@ -653,7 +660,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) // Make sure this call is atomic in the pool. LOCK(pool.cs); UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("loaded", pool.GetLoadTried()); ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); @@ -717,16 +724,18 @@ static RPCHelpMan savemempool() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (!mempool.IsLoaded()) { + if (!mempool.GetLoadTried()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool(mempool)) { + const fs::path& dump_path = MempoolPath(args); + + if (!DumpMempool(mempool, dump_path)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + ret.pushKV("filename", dump_path.u8string()); return ret; }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index ea6db1e9a0..2902b35865 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -476,7 +476,7 @@ static RPCHelpMan prioritisetransaction() static UniValue BIP22ValidationResult(const BlockValidationState& state) { if (state.IsValid()) - return NullUniValue; + return UniValue::VNULL; if (state.IsError()) throw JSONRPCError(RPC_VERIFY_ERROR, state.ToString()); @@ -1040,7 +1040,7 @@ static RPCHelpMan submitheader() BlockValidationState state; chainman.ProcessNewBlockHeaders({h}, state); - if (state.IsValid()) return NullUniValue; + if (state.IsValid()) return UniValue::VNULL; if (state.IsError()) { throw JSONRPCError(RPC_VERIFY_ERROR, state.ToString()); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index fad92629c5..059be61bb4 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -60,7 +60,7 @@ static RPCHelpMan getconnectioncount() NodeContext& node = EnsureAnyNodeContext(request.context); const CConnman& connman = EnsureConnman(node); - return (int)connman.GetNodeCount(ConnectionDirection::Both); + return connman.GetNodeCount(ConnectionDirection::Both); }, }; } @@ -84,7 +84,7 @@ static RPCHelpMan ping() // Request that each node send a ping during next message processing pass peerman.SendPings(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -195,8 +195,9 @@ static RPCHelpMan getpeerinfo() if (stats.m_mapped_as != 0) { obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); } - obj.pushKV("services", strprintf("%016x", stats.nServices)); - obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); + ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; + obj.pushKV("services", strprintf("%016x", services)); + obj.pushKV("servicesnames", GetServicesNames(services)); obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -303,7 +304,7 @@ static RPCHelpMan addnode() { CAddress addr; connman.OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), ConnectionType::MANUAL); - return NullUniValue; + return UniValue::VNULL; } if (strCommand == "add") @@ -319,7 +320,7 @@ static RPCHelpMan addnode() } } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -422,7 +423,7 @@ static RPCHelpMan disconnectnode() throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -639,9 +640,9 @@ static RPCHelpMan getnetworkinfo() obj.pushKV("timeoffset", GetTimeOffset()); if (node.connman) { obj.pushKV("networkactive", node.connman->GetNetworkActive()); - obj.pushKV("connections", (int)node.connman->GetNodeCount(ConnectionDirection::Both)); - obj.pushKV("connections_in", (int)node.connman->GetNodeCount(ConnectionDirection::In)); - obj.pushKV("connections_out", (int)node.connman->GetNodeCount(ConnectionDirection::Out)); + obj.pushKV("connections", node.connman->GetNodeCount(ConnectionDirection::Both)); + obj.pushKV("connections_in", node.connman->GetNodeCount(ConnectionDirection::In)); + obj.pushKV("connections_out", node.connman->GetNodeCount(ConnectionDirection::Out)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -744,7 +745,7 @@ static RPCHelpMan setban() throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously manually banned."); } } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -818,7 +819,7 @@ static RPCHelpMan clearbanned() node.banman->ClearBanned(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -893,7 +894,7 @@ static RPCHelpMan getnodeaddresses() for (const CAddress& addr : vAddr) { UniValue obj(UniValue::VOBJ); - obj.pushKV("time", (int)addr.nTime); + obj.pushKV("time", int64_t{TicksSinceEpoch<std::chrono::seconds>(addr.nTime)}); obj.pushKV("services", (uint64_t)addr.nServices); obj.pushKV("address", addr.ToStringIP()); obj.pushKV("port", addr.GetPort()); @@ -941,7 +942,7 @@ static RPCHelpMan addpeeraddress() if (LookupHost(addr_string, net_addr, false)) { CAddress address{{net_addr, port}, ServiceFlags{NODE_NETWORK | NODE_WITNESS}}; - address.nTime = GetAdjustedTime(); + address.nTime = AdjustedTime(); // The source address is set equal to the address. This is equivalent to the peer // announcing itself. if (node.addrman->Add({address}, address)) { diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 5475662b82..605ebc15a7 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -65,7 +65,7 @@ static RPCHelpMan setmocktime() } } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -85,7 +85,7 @@ static RPCHelpMan invokedisallowedsyscall() throw std::runtime_error("invokedisallowedsyscall is used for testing only."); } TestDisallowedSandboxCall(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -118,7 +118,7 @@ static RPCHelpMan mockscheduler() CHECK_NONFATAL(node_context->scheduler); node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); - return NullUniValue; + return UniValue::VNULL; }, }; } diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index f4bb76f50f..744f809814 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -26,11 +26,6 @@ #include <tuple> #include <vector> -namespace node { -struct NodeContext; -} -using node::NodeContext; - static RPCHelpMan validateaddress() { return RPCHelpMan{ diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 65a7c2963d..7ffb499330 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -46,12 +46,10 @@ #include <univalue.h> using node::AnalyzePSBT; -using node::BroadcastTransaction; using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; -using node::ReadBlockFromDisk; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, CChainState& active_chainstate) { @@ -98,8 +96,8 @@ static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) {RPCResult::Type::NUM, "vout", /*optional=*/true, "The output number (if not coinbase transaction)"}, {RPCResult::Type::OBJ, "scriptSig", /*optional=*/true, "The script (if not coinbase transaction)", { - {RPCResult::Type::STR, "asm", "asm"}, - {RPCResult::Type::STR_HEX, "hex", "hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the signature script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw signature script bytes, hex-encoded"}, }}, {RPCResult::Type::ARR, "txinwitness", /*optional=*/true, "", { @@ -116,9 +114,9 @@ static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) {RPCResult::Type::NUM, "n", "index"}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "the asm"}, + {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "the hex"}, + {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, @@ -694,9 +692,9 @@ const RPCResult decodepsbt_inputs{ {RPCResult::Type::NUM, "amount", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, @@ -708,14 +706,14 @@ const RPCResult decodepsbt_inputs{ {RPCResult::Type::STR, "sighash", /*optional=*/true, "The sighash type to be used"}, {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the redeem script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw redeem script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, }}, {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the witness script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw witness script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, }}, {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", @@ -729,8 +727,8 @@ const RPCResult decodepsbt_inputs{ }}, {RPCResult::Type::OBJ, "final_scriptSig", /*optional=*/true, "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the final signature script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw final signature script bytes, hex-encoded"}, }}, {RPCResult::Type::ARR, "final_scriptwitness", /*optional=*/true, "", { @@ -814,14 +812,14 @@ const RPCResult decodepsbt_outputs{ { {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the redeem script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw redeem script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, }}, {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "asm", "Disassembly of the witness script"}, + {RPCResult::Type::STR_HEX, "hex", "The raw witness script bytes, hex-encoded"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, }}, {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index ca0170c84b..34a4da74f8 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <pubkey.h> +#include <script/miniscript.h> #include <script/script.h> #include <script/standard.h> @@ -161,6 +162,20 @@ public: virtual ~PubkeyProvider() = default; + /** Compare two public keys represented by this provider. + * Used by the Miniscript descriptors to check for duplicate keys in the script. + */ + bool operator<(PubkeyProvider& other) const { + CPubKey a, b; + SigningProvider dummy; + KeyOriginInfo dummy_info; + + GetPubKey(0, dummy, a, dummy_info); + other.GetPubKey(0, dummy, b, dummy_info); + + return a < b; + } + /** Derive a public key. * read_cache is the cache to read keys from (if not nullptr) * write_cache is the cache to write keys to (if not nullptr) @@ -493,12 +508,12 @@ public: /** Base class for all Descriptor implementations. */ class DescriptorImpl : public Descriptor { - //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for Multisig). +protected: + //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for WSH and Multisig). const std::vector<std::unique_ptr<PubkeyProvider>> m_pubkey_args; //! The string name of the descriptor function. const std::string m_name; -protected: //! The sub-descriptor arguments (empty for everything but SH and WSH). //! In doc/descriptors.m this is referred to as SCRIPT expressions sh(SCRIPT) //! and wsh(SCRIPT), and distinct from KEY expressions and ADDR expressions. @@ -563,7 +578,7 @@ public: return true; } - bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const + virtual bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); size_t pos = extra.size() > 0 ? 1 : 0; @@ -917,6 +932,89 @@ public: bool IsSingleType() const final { return true; } }; +/* We instantiate Miniscript here with a simple integer as key type. + * The value of these key integers are an index in the + * DescriptorImpl::m_pubkey_args vector. + */ + +/** + * The context for converting a Miniscript descriptor into a Script. + */ +class ScriptMaker { + //! Keys contained in the Miniscript (the evaluation of DescriptorImpl::m_pubkey_args). + const std::vector<CPubKey>& m_keys; + +public: + ScriptMaker(const std::vector<CPubKey>& keys LIFETIMEBOUND) : m_keys(keys) {} + + std::vector<unsigned char> ToPKBytes(uint32_t key) const { + return {m_keys[key].begin(), m_keys[key].end()}; + } + + std::vector<unsigned char> ToPKHBytes(uint32_t key) const { + auto id = m_keys[key].GetID(); + return {id.begin(), id.end()}; + } +}; + +/** + * The context for converting a Miniscript descriptor to its textual form. + */ +class StringMaker { + //! To convert private keys for private descriptors. + const SigningProvider* m_arg; + //! Keys contained in the Miniscript (a reference to DescriptorImpl::m_pubkey_args). + const std::vector<std::unique_ptr<PubkeyProvider>>& m_pubkeys; + //! Whether to serialize keys as private or public. + bool m_private; + +public: + StringMaker(const SigningProvider* arg LIFETIMEBOUND, const std::vector<std::unique_ptr<PubkeyProvider>>& pubkeys LIFETIMEBOUND, bool priv) + : m_arg(arg), m_pubkeys(pubkeys), m_private(priv) {} + + std::optional<std::string> ToString(uint32_t key) const + { + std::string ret; + if (m_private) { + if (!m_pubkeys[key]->ToPrivateString(*m_arg, ret)) return {}; + } else { + ret = m_pubkeys[key]->ToString(); + } + return ret; + } +}; + +class MiniscriptDescriptor final : public DescriptorImpl +{ +private: + miniscript::NodeRef<uint32_t> m_node; + +protected: + std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, Span<const CScript> scripts, + FlatSigningProvider& provider) const override + { + for (const auto& key : keys) provider.pubkeys.emplace(key.GetID(), key); + return Vector(m_node->ToScript(ScriptMaker(keys))); + } + +public: + MiniscriptDescriptor(std::vector<std::unique_ptr<PubkeyProvider>> providers, miniscript::NodeRef<uint32_t> node) + : DescriptorImpl(std::move(providers), "?"), m_node(std::move(node)) {} + + bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, + const DescriptorCache* cache = nullptr) const override + { + if (const auto res = m_node->ToString(StringMaker(arg, m_pubkey_args, type == StringType::PRIVATE))) { + out = *res; + return true; + } + return false; + } + + bool IsSolvable() const override { return false; } // For now, mark these descriptors as non-solvable (as we don't have signing logic for them). + bool IsSingleType() const final { return true; } +}; + //////////////////////////////////////////////////////////////////////////// // Parser // //////////////////////////////////////////////////////////////////////////// @@ -1058,6 +1156,94 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider)); } +std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) +{ + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); + KeyOriginInfo info; + if (provider.GetKeyOrigin(pubkey.GetID(), info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) +{ + unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; + std::copy(xkey.begin(), xkey.end(), full_key + 1); + CPubKey pubkey(full_key); + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); + KeyOriginInfo info; + if (provider.GetKeyOriginByXOnly(xkey, info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +/** + * The context for parsing a Miniscript descriptor (either from Script or from its textual representation). + */ +struct KeyParser { + //! The Key type is an index in DescriptorImpl::m_pubkey_args + using Key = uint32_t; + //! Must not be nullptr if parsing from string. + FlatSigningProvider* m_out; + //! Must not be nullptr if parsing from Script. + const SigningProvider* m_in; + //! List of keys contained in the Miniscript. + mutable std::vector<std::unique_ptr<PubkeyProvider>> m_keys; + //! Used to detect key parsing errors within a Miniscript. + mutable std::string m_key_parsing_error; + + KeyParser(FlatSigningProvider* out LIFETIMEBOUND, const SigningProvider* in LIFETIMEBOUND) : m_out(out), m_in(in) {} + + bool KeyCompare(const Key& a, const Key& b) const { + return *m_keys.at(a) < *m_keys.at(b); + } + + template<typename I> std::optional<Key> FromString(I begin, I end) const + { + assert(m_out); + Key key = m_keys.size(); + auto pk = ParsePubkey(key, {&*begin, &*end}, ParseScriptContext::P2WSH, *m_out, m_key_parsing_error); + if (!pk) return {}; + m_keys.push_back(std::move(pk)); + return key; + } + + std::optional<std::string> ToString(const Key& key) const + { + return m_keys.at(key)->ToString(); + } + + template<typename I> std::optional<Key> FromPKBytes(I begin, I end) const + { + assert(m_in); + CPubKey pubkey(begin, end); + if (pubkey.IsValid()) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } + + template<typename I> std::optional<Key> FromPKHBytes(I begin, I end) const + { + assert(end - begin == 20); + assert(m_in); + uint160 hash; + std::copy(begin, end, hash.begin()); + CKeyID keyid(hash); + CPubKey pubkey; + if (m_in->GetPubKey(keyid, pubkey)) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } +}; + /** Parse a script in a particular context. */ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { @@ -1279,6 +1465,45 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const error = "Can only have raw() at top level"; return nullptr; } + // Process miniscript expressions. + { + KeyParser parser(&out, nullptr); + auto node = miniscript::FromString(std::string(expr.begin(), expr.end()), parser); + if (node) { + if (ctx != ParseScriptContext::P2WSH) { + error = "Miniscript expressions can only be used in wsh"; + return nullptr; + } + if (parser.m_key_parsing_error != "") { + error = std::move(parser.m_key_parsing_error); + return nullptr; + } + if (!node->IsSane()) { + // Try to find the first insane sub for better error reporting. + auto insane_node = node.get(); + if (const auto sub = node->FindInsaneSub()) insane_node = sub; + if (const auto str = insane_node->ToString(parser)) error = *str; + if (!insane_node->IsValid()) { + error += " is invalid"; + } else { + error += " is not sane"; + if (!insane_node->IsNonMalleable()) { + error += ": malleable witnesses exist"; + } else if (insane_node == node.get() && !insane_node->NeedsSignature()) { + error += ": witnesses without signature exist"; + } else if (!insane_node->CheckTimeLocksMix()) { + error += ": contains mixes of timelocks expressed in blocks and seconds"; + } else if (!insane_node->CheckDuplicateKey()) { + error += ": contains duplicate public keys"; + } else if (!insane_node->ValidSatisfactions()) { + error += ": needs witnesses that may exceed resource limits"; + } + } + return nullptr; + } + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } if (ctx == ParseScriptContext::P2SH) { error = "A function is needed within P2SH"; return nullptr; @@ -1290,29 +1515,6 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const return nullptr; } -std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) -{ - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); - KeyOriginInfo info; - if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - -std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) -{ - unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; - std::copy(xkey.begin(), xkey.end(), full_key + 1); - CPubKey pubkey(full_key); - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); - KeyOriginInfo info; - if (provider.GetKeyOriginByXOnly(xkey, info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { auto match = MatchMultiA(script); @@ -1426,6 +1628,14 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } } + if (ctx == ParseScriptContext::P2WSH) { + KeyParser parser(nullptr, &provider); + auto node = miniscript::FromScript(script, parser); + if (node && node->IsSane()) { + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } + CTxDestination dest; if (ExtractDestination(script, dest)) { if (GetScriptForDestination(dest) == script) { diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 9f56301377..38bb11aad4 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1342,7 +1342,7 @@ public: template <class T> uint256 GetPrevoutsSHA256(const T& txTo) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; for (const auto& txin : txTo.vin) { ss << txin.prevout; } @@ -1353,7 +1353,7 @@ uint256 GetPrevoutsSHA256(const T& txTo) template <class T> uint256 GetSequencesSHA256(const T& txTo) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; for (const auto& txin : txTo.vin) { ss << txin.nSequence; } @@ -1364,7 +1364,7 @@ uint256 GetSequencesSHA256(const T& txTo) template <class T> uint256 GetOutputsSHA256(const T& txTo) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; for (const auto& txout : txTo.vout) { ss << txout; } @@ -1374,7 +1374,7 @@ uint256 GetOutputsSHA256(const T& txTo) /** Compute the (single) SHA256 of the concatenation of all amounts spent by a tx. */ uint256 GetSpentAmountsSHA256(const std::vector<CTxOut>& outputs_spent) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; for (const auto& txout : outputs_spent) { ss << txout.nValue; } @@ -1384,7 +1384,7 @@ uint256 GetSpentAmountsSHA256(const std::vector<CTxOut>& outputs_spent) /** Compute the (single) SHA256 of the concatenation of all scriptPubKeys spent by a tx. */ uint256 GetSpentScriptsSHA256(const std::vector<CTxOut>& outputs_spent) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; for (const auto& txout : outputs_spent) { ss << txout.scriptPubKey; } @@ -1458,9 +1458,9 @@ template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo, template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo); -const CHashWriter HASHER_TAPSIGHASH = TaggedHash("TapSighash"); -const CHashWriter HASHER_TAPLEAF = TaggedHash("TapLeaf"); -const CHashWriter HASHER_TAPBRANCH = TaggedHash("TapBranch"); +const HashWriter HASHER_TAPSIGHASH{TaggedHash("TapSighash")}; +const HashWriter HASHER_TAPLEAF{TaggedHash("TapLeaf")}; +const HashWriter HASHER_TAPBRANCH{TaggedHash("TapBranch")}; static bool HandleMissingData(MissingDataBehavior mdb) { @@ -1499,7 +1499,7 @@ bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, cons return HandleMissingData(mdb); } - CHashWriter ss = HASHER_TAPSIGHASH; + HashWriter ss{HASHER_TAPSIGHASH}; // Epoch static constexpr uint8_t EPOCH = 0; @@ -1544,7 +1544,7 @@ bool SignatureHashSchnorr(uint256& hash_out, ScriptExecutionData& execdata, cons if (output_type == SIGHASH_SINGLE) { if (in_pos >= tx_to.vout.size()) return false; if (!execdata.m_output_hash) { - CHashWriter sha_single_output(SER_GETHASH, 0); + HashWriter sha_single_output{}; sha_single_output << tx_to.vout[in_pos]; execdata.m_output_hash = sha_single_output.GetSHA256(); } @@ -1587,12 +1587,12 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn if ((nHashType & 0x1f) != SIGHASH_SINGLE && (nHashType & 0x1f) != SIGHASH_NONE) { hashOutputs = cacheready ? cache->hashOutputs : SHA256Uint256(GetOutputsSHA256(txTo)); } else if ((nHashType & 0x1f) == SIGHASH_SINGLE && nIn < txTo.vout.size()) { - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; ss << txTo.vout[nIn]; hashOutputs = ss.GetHash(); } - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; // Version ss << txTo.nVersion; // Input prevouts/nSequence (none/all, depending on flags) @@ -1627,7 +1627,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn CTransactionSignatureSerializer<T> txTmp(txTo, scriptCode, nIn, nHashType); // Serialize and hash - CHashWriter ss(SER_GETHASH, 0); + HashWriter ss{}; ss << txTmp << nHashType; return ss.GetHash(); } @@ -1827,7 +1827,7 @@ static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CS uint256 ComputeTapleafHash(uint8_t leaf_version, const CScript& script) { - return (CHashWriter(HASHER_TAPLEAF) << leaf_version << script).GetSHA256(); + return (HashWriter{HASHER_TAPLEAF} << leaf_version << script).GetSHA256(); } uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint256& tapleaf_hash) @@ -1839,7 +1839,7 @@ uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint25 const int path_len = (control.size() - TAPROOT_CONTROL_BASE_SIZE) / TAPROOT_CONTROL_NODE_SIZE; uint256 k = tapleaf_hash; for (int i = 0; i < path_len; ++i) { - CHashWriter ss_branch{HASHER_TAPBRANCH}; + HashWriter ss_branch{HASHER_TAPBRANCH}; Span node{Span{control}.subspan(TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * i, TAPROOT_CONTROL_NODE_SIZE)}; if (std::lexicographical_compare(k.begin(), k.end(), node.begin(), node.end())) { ss_branch << k << node; @@ -1902,7 +1902,7 @@ static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, if (stack.size() >= 2 && !stack.back().empty() && stack.back()[0] == ANNEX_TAG) { // Drop annex (this is non-standard; see IsWitnessStandard) const valtype& annex = SpanPopBack(stack); - execdata.m_annex_hash = (CHashWriter(SER_GETHASH, 0) << annex).GetSHA256(); + execdata.m_annex_hash = (HashWriter{} << annex).GetSHA256(); execdata.m_annex_present = true; } else { execdata.m_annex_present = false; diff --git a/src/script/interpreter.h b/src/script/interpreter.h index adf454aa15..f91578d684 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -233,9 +233,9 @@ static constexpr size_t TAPROOT_CONTROL_NODE_SIZE = 32; static constexpr size_t TAPROOT_CONTROL_MAX_NODE_COUNT = 128; static constexpr size_t TAPROOT_CONTROL_MAX_SIZE = TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * TAPROOT_CONTROL_MAX_NODE_COUNT; -extern const CHashWriter HASHER_TAPSIGHASH; //!< Hasher with tag "TapSighash" pre-fed to it. -extern const CHashWriter HASHER_TAPLEAF; //!< Hasher with tag "TapLeaf" pre-fed to it. -extern const CHashWriter HASHER_TAPBRANCH; //!< Hasher with tag "TapBranch" pre-fed to it. +extern const HashWriter HASHER_TAPSIGHASH; //!< Hasher with tag "TapSighash" pre-fed to it. +extern const HashWriter HASHER_TAPLEAF; //!< Hasher with tag "TapLeaf" pre-fed to it. +extern const HashWriter HASHER_TAPBRANCH; //!< Hasher with tag "TapBranch" pre-fed to it. template <class T> uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn, int nHashType, const CAmount& amount, SigVersion sigversion, const PrecomputedTransactionData* cache = nullptr); diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 2c239c2678..f783d1dafc 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -429,6 +429,21 @@ private: )); } + /** Like TreeEval, but without downfn or State type. + * upfn takes (const Node&, Span<Result>) and returns Result. */ + template<typename Result, typename UpFn> + Result TreeEval(UpFn upfn) const + { + struct DummyState {}; + return std::move(*TreeEvalMaybe<Result>(DummyState{}, + [](DummyState, const Node&, size_t) { return DummyState{}; }, + [&upfn](DummyState, const Node& node, Span<Result> subs) { + Result res{upfn(node, subs)}; + return std::optional<Result>(std::move(res)); + } + )); + } + /** Compare two miniscript subtrees, using a non-recursive algorithm. */ friend int Compare(const Node<Key>& node1, const Node<Key>& node2) { @@ -818,6 +833,15 @@ public: //! Return the expression type. Type GetType() const { return typ; } + //! Find an insane subnode which has no insane children. Nullptr if there is none. + const Node* FindInsaneSub() const { + return TreeEval<const Node*>([](const Node& node, Span<const Node*> subs) -> const Node* { + for (auto& sub: subs) if (sub) return sub; + if (!node.IsSaneSubexpression()) return &node; + return nullptr; + }); + } + //! Check whether this node is valid at all. bool IsValid() const { return !(GetType() == ""_mst) && ScriptSize() <= MAX_STANDARD_P2WSH_SCRIPT_SIZE; } @@ -953,7 +977,11 @@ void BuildBack(const Ctx& ctx, Fragment nt, std::vector<NodeRef<Key>>& construct } } -//! Parse a miniscript from its textual descriptor form. +/** + * Parse a miniscript from its textual descriptor form. + * This does not check whether the script is valid, let alone sane. The caller is expected to use + * the `IsValidTopLevel()` and `IsSaneTopLevel()` to check for these properties on the node. + */ template<typename Key, typename Ctx> inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) { @@ -1255,9 +1283,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) // Sanity checks on the produced miniscript assert(constructed.size() == 1); if (in.size() > 0) return {}; - const NodeRef<Key> tl_node = std::move(constructed.front()); - if (!tl_node->IsValidTopLevel()) return {}; - return tl_node; + return std::move(constructed.front()); } /** Decode a script into opcode/push pairs. diff --git a/src/script/script.h b/src/script/script.h index 3b799ad637..1e5f694d52 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -588,7 +588,6 @@ CScript BuildScript(Ts&&... inputs) int cnt{0}; ([&ret, &cnt] (Ts&& input) { - cnt++; if constexpr (std::is_same_v<std::remove_cv_t<std::remove_reference_t<Ts>>, CScript>) { // If it is a CScript, extend ret with it. Move or copy the first element instead. if (cnt == 0) { @@ -600,6 +599,7 @@ CScript BuildScript(Ts&&... inputs) // Otherwise invoke CScript::operator<<. ret << input; } + cnt++; } (std::forward<Ts>(inputs)), ...); return ret; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index a3681d26cc..3b8071d9d1 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -165,7 +165,7 @@ static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatu if (leaf_version != TAPROOT_LEAF_TAPSCRIPT) return false; SigVersion sigversion = SigVersion::TAPSCRIPT; - uint256 leaf_hash = (CHashWriter(HASHER_TAPLEAF) << uint8_t(leaf_version) << script).GetSHA256(); + uint256 leaf_hash = (HashWriter{HASHER_TAPLEAF} << uint8_t(leaf_version) << script).GetSHA256(); // <xonly pubkey> OP_CHECKSIG if (script.size() == 34 && script[33] == OP_CHECKSIG && script[0] == 0x20) { diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 5d80891485..b3f6a1b669 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -375,9 +375,9 @@ bool IsValidDestination(const CTxDestination& dest) { } /* Lexicographically sort a and b's hash, and compute parent hash. */ if (a.hash < b.hash) { - ret.hash = (CHashWriter(HASHER_TAPBRANCH) << a.hash << b.hash).GetSHA256(); + ret.hash = (HashWriter{HASHER_TAPBRANCH} << a.hash << b.hash).GetSHA256(); } else { - ret.hash = (CHashWriter(HASHER_TAPBRANCH) << b.hash << a.hash).GetSHA256(); + ret.hash = (HashWriter{HASHER_TAPBRANCH} << b.hash << a.hash).GetSHA256(); } return ret; } @@ -452,7 +452,7 @@ TaprootBuilder& TaprootBuilder::Add(int depth, const CScript& script, int leaf_v if (!IsValid()) return *this; /* Construct NodeInfo object with leaf hash and (if track is true) also leaf information. */ NodeInfo node; - node.hash = (CHashWriter{HASHER_TAPLEAF} << uint8_t(leaf_version) << script).GetSHA256(); + node.hash = (HashWriter{HASHER_TAPLEAF} << uint8_t(leaf_version) << script).GetSHA256(); if (track) node.leaves.emplace_back(LeafInfo{script, leaf_version, {}}); /* Insert into the branch. */ Insert(std::move(node), depth); @@ -610,7 +610,7 @@ std::optional<std::vector<std::tuple<int, CScript, int>>> InferTaprootTree(const node.done = true; stack.pop_back(); } else if (node.sub[0]->done && !node.sub[1]->done && !node.sub[1]->explored && !node.sub[1]->hash.IsNull() && - (CHashWriter{HASHER_TAPBRANCH} << node.sub[1]->hash << node.sub[1]->hash).GetSHA256() == node.hash) { + (HashWriter{HASHER_TAPBRANCH} << node.sub[1]->hash << node.sub[1]->hash).GetSHA256() == node.hash) { // Whenever there are nodes with two identical subtrees under it, we run into a problem: // the control blocks for the leaves underneath those will be identical as well, and thus // they will all be matched to the same path in the tree. The result is that at the location diff --git a/src/serialize.h b/src/serialize.h index a1cce78451..89a9f32240 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -520,6 +520,29 @@ struct CompactSizeFormatter } }; +template <typename U, bool LOSSY = false> +struct ChronoFormatter { + template <typename Stream, typename Tp> + void Unser(Stream& s, Tp& tp) + { + U u; + s >> u; + // Lossy deserialization does not make sense, so force Wnarrowing + tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}}; + } + template <typename Stream, typename Tp> + void Ser(Stream& s, Tp tp) + { + if constexpr (LOSSY) { + s << U(tp.time_since_epoch().count()); + } else { + s << U{tp.time_since_epoch().count()}; + } + } +}; +template <typename U> +using LossyChronoFormatter = ChronoFormatter<U, true>; + class CompactSizeWriter { protected: diff --git a/src/streams.h b/src/streams.h index 96b7696f72..f14d347380 100644 --- a/src/streams.h +++ b/src/streams.h @@ -465,35 +465,28 @@ public: }; - /** Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. * If you're returning the file pointer, return file.release(). * If you need to close the file early, use file.fclose() instead of fclose(file). */ -class CAutoFile +class AutoFile { -private: - const int nType; - const int nVersion; - +protected: FILE* file; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn) - { - file = filenew; - } + explicit AutoFile(FILE* filenew) : file{filenew} {} - ~CAutoFile() + ~AutoFile() { fclose(); } // Disallow copies - CAutoFile(const CAutoFile&) = delete; - CAutoFile& operator=(const CAutoFile&) = delete; + AutoFile(const AutoFile&) = delete; + AutoFile& operator=(const AutoFile&) = delete; void fclose() { @@ -504,14 +497,14 @@ public: } /** Get wrapped FILE* with transfer of ownership. - * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller + * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller * of this function to clean up the returned FILE*. */ FILE* release() { FILE* ret = file; file = nullptr; return ret; } /** Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the - * CAutoFile outlives use of the passed pointer. + * AutoFile outlives use of the passed pointer. */ FILE* Get() const { return file; } @@ -522,40 +515,62 @@ public: // // Stream subset // - int GetType() const { return nType; } - int GetVersion() const { return nVersion; } - void read(Span<std::byte> dst) { - if (!file) - throw std::ios_base::failure("CAutoFile::read: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { - throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); } } void ignore(size_t nSize) { - if (!file) - throw std::ios_base::failure("CAutoFile::ignore: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); unsigned char data[4096]; while (nSize > 0) { size_t nNow = std::min<size_t>(nSize, sizeof(data)); if (fread(data, 1, nNow, file) != nNow) - throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed"); + throw std::ios_base::failure(feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); nSize -= nNow; } } void write(Span<const std::byte> src) { - if (!file) - throw std::ios_base::failure("CAutoFile::write: file handle is nullptr"); + if (!file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); if (fwrite(src.data(), 1, src.size(), file) != src.size()) { - throw std::ios_base::failure("CAutoFile::write: write failed"); + throw std::ios_base::failure("AutoFile::write: write failed"); } } + template <typename T> + AutoFile& operator<<(const T& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator<<: file handle is nullptr"); + ::Serialize(*this, obj); + return *this; + } + + template <typename T> + AutoFile& operator>>(T&& obj) + { + if (!file) throw std::ios_base::failure("AutoFile::operator>>: file handle is nullptr"); + ::Unserialize(*this, obj); + return *this; + } +}; + +class CAutoFile : public AutoFile +{ +private: + const int nType; + const int nVersion; + +public: + CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} + int GetType() const { return nType; } + int GetVersion() const { return nVersion; } + template<typename T> CAutoFile& operator<<(const T& obj) { diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 12cf1176a6..b1372a3e98 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -225,7 +225,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) { auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CAddress addr{CAddress(ResolveService("253.3.3.3", 8333), NODE_NONE)}; - int64_t start_time{GetAdjustedTime()}; + const auto start_time{AdjustedTime()}; addr.nTime = start_time; // test that multiplicity stays at 1 if nTime doesn't increase @@ -244,7 +244,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) for (unsigned int i = 1; i < 400; ++i) { std::string addr_ip{ToString(i % 256) + "." + ToString(i >> 8 % 256) + ".1.1"}; CNetAddr source{ResolveIP(addr_ip)}; - addr.nTime = start_time + i; + addr.nTime = start_time + std::chrono::seconds{i}; addrman->Add({addr}, source); } AddressPosition addr_pos_multi = addrman->FindAddressEntry(addr).value(); @@ -295,15 +295,15 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK_EQUAL(vAddr1.size(), 0U); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); - addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false + addr1.nTime = AdjustedTime(); // Set time so isTerrible = false CAddress addr2 = CAddress(ResolveService("250.251.2.2", 9999), NODE_NONE); - addr2.nTime = GetAdjustedTime(); + addr2.nTime = AdjustedTime(); CAddress addr3 = CAddress(ResolveService("251.252.2.3", 8333), NODE_NONE); - addr3.nTime = GetAdjustedTime(); + addr3.nTime = AdjustedTime(); CAddress addr4 = CAddress(ResolveService("252.253.3.4", 8333), NODE_NONE); - addr4.nTime = GetAdjustedTime(); + addr4.nTime = AdjustedTime(); CAddress addr5 = CAddress(ResolveService("252.254.4.5", 8333), NODE_NONE); - addr5.nTime = GetAdjustedTime(); + addr5.nTime = AdjustedTime(); CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.2.3.3"); @@ -329,7 +329,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CAddress addr = CAddress(ResolveService(strAddr), NODE_NONE); // Ensure that for all addrs in addrman, isTerrible == false. - addr.nTime = GetAdjustedTime(); + addr.nTime = AdjustedTime(); addrman->Add({addr}, ResolveIP(strAddr)); if (i % 8 == 0) addrman->Good(addr); @@ -821,8 +821,8 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) // Ensure test of address fails, so that it is evicted. // Update entry in tried by setting last good connection in the deep past. - BOOST_CHECK(!addrman->Good(info, /*nTime=*/1)); - addrman->Attempt(info, /*fCountFailure=*/false, /*nTime=*/GetAdjustedTime() - 61); + BOOST_CHECK(!addrman->Good(info, NodeSeconds{1s})); + addrman->Attempt(info, /*fCountFailure=*/false, AdjustedTime() - 61s); // Should swap 36 for 19. addrman->ResolveCollisions(); @@ -966,7 +966,7 @@ BOOST_AUTO_TEST_CASE(addrman_update_address) CNetAddr source{ResolveIP("252.2.2.2")}; CAddress addr{CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE)}; - int64_t start_time{GetAdjustedTime() - 10000}; + const auto start_time{AdjustedTime() - 10000s}; addr.nTime = start_time; BOOST_CHECK(addrman->Add({addr}, source)); BOOST_CHECK_EQUAL(addrman->size(), 1U); @@ -978,7 +978,7 @@ BOOST_AUTO_TEST_CASE(addrman_update_address) addrman->SetServices(addr_diff_port, NODE_NETWORK_LIMITED); std::vector<CAddress> vAddr1{addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt)}; BOOST_CHECK_EQUAL(vAddr1.size(), 1U); - BOOST_CHECK_EQUAL(vAddr1.at(0).nTime, start_time); + BOOST_CHECK(vAddr1.at(0).nTime == start_time); BOOST_CHECK_EQUAL(vAddr1.at(0).nServices, NODE_NONE); // Updating an addrman entry with the correct port is successful @@ -986,7 +986,7 @@ BOOST_AUTO_TEST_CASE(addrman_update_address) addrman->SetServices(addr, NODE_NETWORK_LIMITED); std::vector<CAddress> vAddr2 = addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); BOOST_CHECK_EQUAL(vAddr2.size(), 1U); - BOOST_CHECK(vAddr2.at(0).nTime >= start_time + 10000); + BOOST_CHECK(vAddr2.at(0).nTime >= start_time + 10000s); BOOST_CHECK_EQUAL(vAddr2.at(0).nServices, NODE_NETWORK_LIMITED); } diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index c31e4e51f7..1a182209b8 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/merkle.h> #include <consensus/validation.h> #include <index/blockfilterindex.h> +#include <interfaces/chain.h> #include <node/miner.h> #include <pow.h> #include <script/standard.h> @@ -110,7 +111,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { - BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); + BlockFilterIndex filter_index(interfaces::MakeChain(m_node), BlockFilterType::BASIC, 1 << 20, true); uint256 last_header; @@ -137,7 +138,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // BlockUntilSyncedToCurrentChain should return false before index is started. BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(filter_index.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(filter_index.Start()); // Allow filter index to catch up with the block index. constexpr int64_t timeout_ms = 10 * 1000; @@ -279,14 +280,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); BOOST_CHECK(filter_index == nullptr); - BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); BOOST_CHECK(filter_index != nullptr); BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC); // Initialize returns false if index already exists. - BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(!InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); int iter_count = 0; ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; }); @@ -301,7 +302,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) BOOST_CHECK(filter_index == nullptr); // Reinitialize index. - BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); DestroyAllBlockFilterIndexes(); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 50eb479035..c93d05a93b 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <index/coinstatsindex.h> +#include <interfaces/chain.h> #include <test/util/setup_common.h> #include <test/util/validation.h> #include <util/time.h> @@ -13,9 +14,6 @@ #include <chrono> -using kernel::CCoinsStats; -using kernel::CoinStatsHashType; - BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) static void IndexWaitSynced(BaseIndex& index) @@ -31,7 +29,7 @@ static void IndexWaitSynced(BaseIndex& index) BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) { - CoinStatsIndex coin_stats_index{1 << 20, true}; + CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true}; const CBlockIndex* block_index; { @@ -46,7 +44,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) // is started. BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(coin_stats_index.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(coin_stats_index.Start()); IndexWaitSynced(coin_stats_index); @@ -90,8 +88,8 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) CChainState& chainstate = Assert(m_node.chainman)->ActiveChainstate(); const CChainParams& params = Params(); { - CoinStatsIndex index{1 << 20}; - BOOST_REQUIRE(index.Start(chainstate)); + CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; + BOOST_REQUIRE(index.Start()); IndexWaitSynced(index); std::shared_ptr<const CBlock> new_block; CBlockIndex* new_block_index = nullptr; @@ -116,9 +114,9 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) } { - CoinStatsIndex index{1 << 20}; + CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; // Make sure the index can be loaded. - BOOST_REQUIRE(index.Start(chainstate)); + BOOST_REQUIRE(index.Start()); index.Stop(); } } diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index c87ed82c88..66c4605afa 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -15,6 +15,7 @@ #include <serialize.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <timedata.h> #include <util/string.h> #include <util/system.h> #include <util/time.h> @@ -44,17 +45,15 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) // work. BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); + ConnmanTestMsg& connman = static_cast<ConnmanTestMsg&>(*m_node.connman); // Disable inactivity checks for this test to avoid interference - static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999s); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, - *m_node.chainman, *m_node.mempool, false); + connman.SetPeerConnectTimeout(99999s); + PeerManager& peerman = *m_node.peerman; // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); NodeId id{0}; CNode dummyNode1{id++, - ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, addr1, /*nKeyedNetGroupIn=*/0, @@ -63,10 +62,16 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) /*addrNameIn=*/"", ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false}; - dummyNode1.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode1); - dummyNode1.fSuccessfullyConnected = true; + connman.Handshake( + /*node=*/dummyNode1, + /*successfully_connected=*/true, + /*remote_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*local_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*permission_flags=*/NetPermissionFlags::None, + /*version=*/PROTOCOL_VERSION, + /*relay_txs=*/true); + TestOnlyResetTimeData(); // This test requires that we have a chain with non-zero work. { @@ -78,7 +83,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Test starts here { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in getheaders } { LOCK(dummyNode1.cs_vSend); @@ -91,7 +96,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) SetMockTime(nStartTime+21*60); { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in getheaders } { LOCK(dummyNode1.cs_vSend); @@ -101,18 +106,17 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) SetMockTime(nStartTime+24*60); { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in disconnect + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in disconnect } BOOST_CHECK(dummyNode1.fDisconnect == true); - peerLogic->FinalizeNode(dummyNode1); + peerman.FinalizeNode(dummyNode1); } static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode{id++, - ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -124,7 +128,7 @@ static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerM CNode &node = *vNodes.back(); node.SetCommonVersion(PROTOCOL_VERSION); - peerLogic.InitializeNode(&node); + peerLogic.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); node.fSuccessfullyConnected = true; connman.AddTestNode(node); @@ -292,7 +296,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) banman->ClearBanned(); NodeId id{0}; nodes[0] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[0], /*nKeyedNetGroupIn=*/0, @@ -302,7 +305,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[0]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[0]); + peerLogic->InitializeNode(*nodes[0], NODE_NETWORK); nodes[0]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[0]); peerLogic->UnitTestMisbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged @@ -315,7 +318,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(!banman->IsDiscouraged(other_addr)); // Different address, not discouraged nodes[1] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[1], /*nKeyedNetGroupIn=*/1, @@ -325,7 +327,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[1]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[1]); + peerLogic->InitializeNode(*nodes[1], NODE_NETWORK); nodes[1]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[1]); peerLogic->UnitTestMisbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1); @@ -353,7 +355,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) // Make sure non-IP peers are discouraged and disconnected properly. nodes[2] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[2], /*nKeyedNetGroupIn=*/1, @@ -363,7 +364,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false}; nodes[2]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[2]); + peerLogic->InitializeNode(*nodes[2], NODE_NETWORK); nodes[2]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[2]); peerLogic->UnitTestMisbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD); @@ -398,7 +399,6 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) CAddress addr(ip(0xa0b0c001), NODE_NONE); NodeId id{0}; CNode dummyNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/4, @@ -408,7 +408,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) ConnectionType::INBOUND, /*inbound_onion=*/false}; dummyNode.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode); + peerLogic->InitializeNode(dummyNode, NODE_NETWORK); dummyNode.fSuccessfullyConnected = true; peerLogic->UnitTestMisbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 63c86a896d..a8c666079d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -141,14 +141,13 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& } else { parse_priv = Parse(prv, keys_priv, error); } + BOOST_CHECK_MESSAGE(parse_priv, error); if (replace_apostrophe_with_h_in_pub) { parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); } else { parse_pub = Parse(pub, keys_pub, error); } - - BOOST_CHECK(parse_priv); - BOOST_CHECK(parse_pub); + BOOST_CHECK_MESSAGE(parse_pub, error); // Check that the correct OutputType is inferred BOOST_CHECK(parse_priv->GetOutputType() == type); @@ -161,8 +160,8 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& // Check that both versions serialize back to the public version. std::string pub1 = parse_priv->ToString(); std::string pub2 = parse_pub->ToString(); - BOOST_CHECK(EqualDescriptor(pub, pub1)); - BOOST_CHECK(EqualDescriptor(pub, pub2)); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub1), "Private ser: " + pub1 + " Public desc: " + pub); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub2), "Public ser: " + pub2 + " Public desc: " + pub); // Check that both can be serialized with private key back to the private version, but not without private key. if (!(flags & MISSING_PRIVKEYS)) { @@ -486,6 +485,29 @@ Check("sh(wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGN } nonminimalmultisig << std::vector<unsigned char>{4} << OP_CHECKMULTISIG; CheckInferRaw(nonminimalmultisig); + + // Miniscript tests + + // Invalid checksum + CheckUnparsable("wsh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))#abcdef12", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))#abcdef12", "Provided checksum 'abcdef12' does not match computed checksum 'tyzp6a7p'"); + // Only p2wsh context is valid + CheckUnparsable("sh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("tr(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "tr(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "tr(): key 'and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10))' is not valid"); + CheckUnparsable("raw(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("", "tr(034D2224bbbbbbbbbbcbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb40,{{{{{{{{{{{{{{{{{{{{{{multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008')", "'multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008'' is not a valid descriptor function"); + // Insane at top level + CheckUnparsable("wsh(and_b(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)) is invalid"); + // Invalid sub + CheckUnparsable("wsh(and_v(vc:andor(v:pk_k(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_v(vc:andor(v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204) is invalid"); + // Insane subs + CheckUnparsable("wsh(or_i(older(1),pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: witnesses without signature exist"); + CheckUnparsable("wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: malleable witnesses exist"); + CheckUnparsable("wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(older(1),a:older(100000000)) is not sane: contains mixes of timelocks expressed in blocks and seconds"); + CheckUnparsable("wsh(and_b(or_b(pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),s:pk(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: contains duplicate public keys"); + // Valid with extended keys. + Check("wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", UNSOLVABLE, {{"0020acf425291b98a1d7e0d4690139442abc289175be32ef1f75945e339924246d73"}}, OutputType::BECH32, {{},{0}}); + // Valid under sh(wsh()) and with a mix of xpubs and raw keys + Check("sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", UNSOLVABLE | MIXED_PUBKEYS, {{"a914767e9119ff3b3ac0cb6dcfe21de1842ccf85f1c487"}}, OutputType::P2SH_SEGWIT, {{},{0}}); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index d54d6b6471..605faa08e4 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -41,26 +41,26 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Write first line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1))}; file << LIMITED_STRING(line1, 256); } // Attempt to append to file opened in read-only mode. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2), true)}; BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); } // Append second line to file. { - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file << LIMITED_STRING(line2, 256); } // Read text from file in read-only mode. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos1), true)}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line1); @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Read text from file with position offset. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(0, pos2))}; file >> LIMITED_STRING(text, 256); BOOST_CHECK_EQUAL(text, line2); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) // Ensure another file in the sequence has no data. { std::string text; - CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + AutoFile file{seq.Open(FlatFilePos(1, pos2))}; BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); } } diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index af7a282781..7668940cbc 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -113,11 +113,11 @@ void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) for (size_t j = 0; j < num_addresses; ++j) { const auto addr = CAddress{CService{RandAddr(fuzzed_data_provider, fast_random_context), 8333}, NODE_NETWORK}; - const auto time_penalty = fast_random_context.randrange(100000001); + const std::chrono::seconds time_penalty{fast_random_context.randrange(100000001)}; addrman.Add({addr}, source, time_penalty); if (n > 0 && addrman.size() % n == 0) { - addrman.Good(addr, GetTime()); + addrman.Good(addr, Now<NodeSeconds>()); } // Add 10% of the addresses from more than one source. @@ -161,7 +161,7 @@ public: CSipHasher hasher(0, 0); auto addr_key = a.GetKey(); auto source_key = a.source.GetAddrBytes(); - hasher.Write(a.nLastSuccess); + hasher.Write(TicksSinceEpoch<std::chrono::seconds>(a.m_last_success)); hasher.Write(a.nAttempts); hasher.Write(a.nRefCount); hasher.Write(a.fInTried); @@ -175,8 +175,8 @@ public: }; auto addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { - return std::tie(static_cast<const CService&>(lhs), lhs.source, lhs.nLastSuccess, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == - std::tie(static_cast<const CService&>(rhs), rhs.source, rhs.nLastSuccess, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); + return std::tie(static_cast<const CService&>(lhs), lhs.source, lhs.m_last_success, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == + std::tie(static_cast<const CService&>(rhs), rhs.source, rhs.m_last_success, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); }; using Addresses = std::unordered_set<AddrInfo, decltype(addrinfo_hasher), decltype(addrinfo_eq)>; @@ -269,25 +269,25 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); if (opt_net_addr) { - addr_man.Add(addresses, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); + addr_man.Add(addresses, *opt_net_addr, std::chrono::seconds{ConsumeTime(fuzzed_data_provider, 0, 100000000)}); } }, [&] { const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); if (opt_service) { - addr_man.Good(*opt_service, ConsumeTime(fuzzed_data_provider)); + addr_man.Good(*opt_service, NodeSeconds{std::chrono::seconds{ConsumeTime(fuzzed_data_provider)}}); } }, [&] { const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); if (opt_service) { - addr_man.Attempt(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); + addr_man.Attempt(*opt_service, fuzzed_data_provider.ConsumeBool(), NodeSeconds{std::chrono::seconds{ConsumeTime(fuzzed_data_provider)}}); } }, [&] { const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); if (opt_service) { - addr_man.Connected(*opt_service, ConsumeTime(fuzzed_data_provider)); + addr_man.Connected(*opt_service, NodeSeconds{std::chrono::seconds{ConsumeTime(fuzzed_data_provider)}}); } }, [&] { diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 3b410930ed..1a8957d090 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile auto_file = fuzzed_auto_file_provider.open(); + AutoFile auto_file{fuzzed_auto_file_provider.open()}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, @@ -53,8 +53,6 @@ FUZZ_TARGET(autofile) }); } (void)auto_file.Get(); - (void)auto_file.GetType(); - (void)auto_file.GetVersion(); (void)auto_file.IsNull(); if (fuzzed_data_provider.ConsumeBool()) { FILE* f = auto_file.release(); diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index 8c0ed32d51..01edb06138 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -23,7 +23,7 @@ FUZZ_TARGET(chain) disk_block_index->phashBlock = &zero; { LOCK(::cs_main); - (void)disk_block_index->GetBlockHash(); + (void)disk_block_index->ConstructBlockHash(); (void)disk_block_index->GetBlockPos(); (void)disk_block_index->GetBlockTime(); (void)disk_block_index->GetBlockTimeMax(); @@ -31,7 +31,6 @@ FUZZ_TARGET(chain) (void)disk_block_index->GetUndoPos(); (void)disk_block_index->HaveTxsDownloaded(); (void)disk_block_index->IsValid(); - (void)disk_block_index->ToString(); } const CBlockHeader block_header = disk_block_index->GetBlockHeader(); diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index bfa977520b..f4b7dc08fd 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -31,6 +31,13 @@ FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) if (fuzzed_block_file == nullptr) { return; } - FlatFilePos flat_file_pos; - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); + if (fuzzed_data_provider.ConsumeBool()) { + // Corresponds to the -reindex case (track orphan blocks across files). + FlatFilePos flat_file_pos; + std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; + g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file, &flat_file_pos, &blocks_with_unknown_parent); + } else { + // Corresponds to the -loadblock= case (orphan blocks aren't tracked across files). + g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file); + } } diff --git a/src/test/fuzz/mempool_utils.h b/src/test/fuzz/mempool_utils.h new file mode 100644 index 0000000000..bfe12e30ba --- /dev/null +++ b/src/test/fuzz/mempool_utils.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +#define BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H + +#include <validation.h> + +class DummyChainState final : public CChainState +{ +public: + void SetMempool(CTxMemPool* mempool) + { + m_mempool = mempool; + } +}; + +#endif // BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 4981287152..741810f6a2 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -68,7 +68,6 @@ FUZZ_TARGET_INIT(net, initialize_net) (void)node.GetAddrLocal(); (void)node.GetId(); (void)node.GetLocalNonce(); - (void)node.GetLocalServices(); const int ref_count = node.GetRefCount(); assert(ref_count >= 0); (void)node.GetCommonVersion(); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index c7a76aa52f..0cc210f26f 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) return ParseNonRFCJSONValue(random_string); } catch (const std::runtime_error&) { valid = false; - return NullUniValue; + return UniValue{}; } }(); if (!valid) { diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 58c19a91cb..637ba503c6 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -76,7 +76,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) } { FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; block_policy_estimator.Write(fuzzed_auto_file); block_policy_estimator.Read(fuzzed_auto_file); } diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 77402c260a..436873c955 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - CAutoFile fuzzed_auto_file = fuzzed_auto_file_provider.open(); + AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args)}; if (block_policy_estimator.Read(fuzzed_auto_file)) { diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 2d88ee295b..cfb112879a 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -8,6 +8,7 @@ #include <node/miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/script.h> @@ -34,15 +35,6 @@ struct MockedTxPool : public CTxMemPool { } }; -class DummyChainState final : public CChainState -{ -public: - void SetMempool(CTxMemPool* mempool) - { - m_mempool = mempool; - } -}; - void initialize_tx_pool() { static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); @@ -144,7 +136,6 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) auto& chainstate{static_cast<DummyChainState&>(node.chainman->ActiveChainstate())}; MockTime(fuzzed_data_provider, chainstate); - SetMempoolConstraints(*node.args, fuzzed_data_provider); // All RBF-spendable outpoints std::set<COutPoint> outpoints_rbf; @@ -158,6 +149,7 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) // The sum of the values of all spendable outpoints constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN}; + SetMempoolConstraints(*node.args, fuzzed_data_provider); CTxMemPool tx_pool_{MakeMempool(node)}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); @@ -229,9 +221,6 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) MockTime(fuzzed_data_provider, chainstate); } if (fuzzed_data_provider.ConsumeBool()) { - SetMempoolConstraints(*node.args, fuzzed_data_provider); - } - if (fuzzed_data_provider.ConsumeBool()) { tx_pool.RollingFeeUpdate(); } if (fuzzed_data_provider.ConsumeBool()) { @@ -324,7 +313,6 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) auto& chainstate = node.chainman->ActiveChainstate(); MockTime(fuzzed_data_provider, chainstate); - SetMempoolConstraints(*node.args, fuzzed_data_provider); std::vector<uint256> txids; for (const auto& outpoint : g_outpoints_coinbase_init_mature) { @@ -336,6 +324,7 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) txids.push_back(ConsumeUInt256(fuzzed_data_provider)); } + SetMempoolConstraints(*node.args, fuzzed_data_provider); CTxMemPool tx_pool_{MakeMempool(node)}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); @@ -347,9 +336,6 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) MockTime(fuzzed_data_provider, chainstate); } if (fuzzed_data_provider.ConsumeBool()) { - SetMempoolConstraints(*node.args, fuzzed_data_provider); - } - if (fuzzed_data_provider.ConsumeBool()) { tx_pool.RollingFeeUpdate(); } if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index d318baa6a2..7580651371 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -3,8 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/amount.h> -#include <net.h> +#include <consensus/validation.h> #include <net_processing.h> +#include <node/eviction.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> #include <sync.h> @@ -65,7 +67,7 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) for (uint32_t i = 0; i < num_out; i++) { tx_mut.vout.emplace_back(CAmount{0}, CScript{}); } - // restore previously poped outpoints + // restore previously popped outpoints for (auto& in : tx_mut.vin) { outpoints.push_back(in.prevout); } @@ -99,16 +101,20 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) [&] { bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); // AddTx should return false if tx is too big or already have it + // tx weight is unknown, we only check when tx is already in orphanage { LOCK(g_cs_orphans); - Assert(have_tx != orphanage.AddTx(tx, peer_id)); + bool add_tx = orphanage.AddTx(tx, peer_id); + // have_tx == true -> add_tx == false + Assert(!have_tx || !add_tx); } have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); - // tx should already be added since it will not be too big in the test - // have_tx should be true and AddTx should fail { LOCK(g_cs_orphans); - Assert(have_tx && !orphanage.AddTx(tx, peer_id)); + bool add_tx = orphanage.AddTx(tx, peer_id); + // if have_tx is still false, it must be too big + Assert(!have_tx == (GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT)); + Assert(!have_tx || !add_tx); } }, [&] { @@ -132,10 +138,8 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) [&] { // test mocktime and expiry SetMockTime(ConsumeTime(fuzzed_data_provider)); - auto size_before = orphanage.Size(); auto limit = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - auto n_evicted = WITH_LOCK(g_cs_orphans, return orphanage.LimitOrphans(limit)); - Assert(size_before - n_evicted <= limit); + WITH_LOCK(g_cs_orphans, orphanage.LimitOrphans(limit)); Assert(orphanage.Size() <= limit); }); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index f0cff74f94..ba1a634e41 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -294,6 +294,7 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, connman.Handshake(node, /*successfully_connected=*/fuzzed_data_provider.ConsumeBool(), /*remote_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), + /*local_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), /*permission_flags=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS), /*version=*/fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()), /*relay_txs=*/fuzzed_data_provider.ConsumeBool()); @@ -526,6 +527,11 @@ CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept return net_addr; } +CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeService(fuzzed_data_provider), ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), NodeSeconds{std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<uint32_t>()}}}; +} + FILE* FuzzedFileProvider::open() { SetFuzzedErrNo(m_fuzzed_data_provider); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index d189a50a51..33d9ab3cc3 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,7 +8,7 @@ #include <arith_uint256.h> #include <chainparamsbase.h> #include <coins.h> -#include <compat.h> +#include <compat/compat.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <merkleblock.h> @@ -287,16 +287,12 @@ inline CService ConsumeService(FuzzedDataProvider& fuzzed_data_provider) noexcep return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint16_t>()}; } -inline CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {ConsumeService(fuzzed_data_provider), ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; -} +CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept; template <bool ReturnUniquePtr = false> auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept { const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegralInRange<NodeId>(0, std::numeric_limits<NodeId>::max())); - const ServiceFlags local_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); const auto sock = std::make_shared<FuzzedSock>(fuzzed_data_provider); const CAddress address = ConsumeAddress(fuzzed_data_provider); const uint64_t keyed_net_group = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); @@ -307,7 +303,6 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N const bool inbound_onion{conn_type == ConnectionType::INBOUND ? fuzzed_data_provider.ConsumeBool() : false}; if constexpr (ReturnUniquePtr) { return std::make_unique<CNode>(node_id, - local_services, sock, address, keyed_net_group, @@ -318,7 +313,6 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N inbound_onion); } else { return CNode{node_id, - local_services, sock, address, keyed_net_group, @@ -361,17 +355,16 @@ public: class FuzzedAutoFileProvider { - FuzzedDataProvider& m_fuzzed_data_provider; FuzzedFileProvider m_fuzzed_file_provider; public: - FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider{fuzzed_data_provider}, m_fuzzed_file_provider{fuzzed_data_provider} + FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_file_provider{fuzzed_data_provider} { } - CAutoFile open() + AutoFile open() { - return {m_fuzzed_file_provider.open(), m_fuzzed_data_provider.ConsumeIntegral<int>(), m_fuzzed_data_provider.ConsumeIntegral<int>()}; + return AutoFile{m_fuzzed_file_provider.open()}; } }; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 33496a457e..0b596492be 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -39,13 +39,13 @@ FUZZ_TARGET_INIT(utxo_snapshot, initialize_chain) Assert(!chainman.SnapshotBlockhash()); { - CAutoFile outfile{fsbridge::fopen(snapshot_path, "wb"), SER_DISK, CLIENT_VERSION}; + AutoFile outfile{fsbridge::fopen(snapshot_path, "wb")}; const auto file_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; outfile << Span{file_data}; } const auto ActivateFuzzedSnapshot{[&] { - CAutoFile infile{fsbridge::fopen(snapshot_path, "rb"), SER_DISK, CLIENT_VERSION}; + AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; SnapshotMetadata metadata; try { infile >> metadata; diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 9532610f8d..90c1a71d9f 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,10 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <kernel/mempool_persist.h> + #include <chainparamsbase.h> #include <mempool_args.h> +#include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -15,6 +19,10 @@ #include <cstdint> #include <vector> +using kernel::DumpMempool; + +using node::MempoolPath; + namespace { const TestingSetup* g_setup; } // namespace @@ -33,9 +41,12 @@ FUZZ_TARGET_INIT(validation_load_mempool, initialize_validation_load_mempool) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; + chainstate.SetMempool(&pool); + auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)LoadMempool(pool, g_setup->m_node.chainman->ActiveChainstate(), fuzzed_fopen); - (void)DumpMempool(pool, fuzzed_fopen, true); + (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 3877fea907..95e8476b77 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -111,11 +111,17 @@ struct KeyConverter { assert(it != g_testdata->pkmap.end()); return it->second; } + + std::optional<std::string> ToString(const Key& key) const { + return HexStr(ToPKBytes(key)); + } }; //! Singleton instance of KeyConverter. const KeyConverter CONVERTER{}; +// https://github.com/llvm/llvm-project/issues/53444 +// NOLINTNEXTLINE(misc-unused-using-decls) using miniscript::operator"" _mst; enum TestMode : int { @@ -276,7 +282,7 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // (for now) have 'd:' be 'u'. This tests we can't use a 'd:' wrapper for a thresh, which requires // its subs to all be 'u' (taken from https://github.com/rust-bitcoin/rust-miniscript/discussions/341). const auto ms_minimalif = miniscript::FromString("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),sc:pk_k(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798),sdv:older(32))", CONVERTER); - BOOST_CHECK(!ms_minimalif); + BOOST_CHECK(ms_minimalif && !ms_minimalif->IsValid()); // A Miniscript with duplicate keys is not sane const auto ms_dup1 = miniscript::FromString("and_v(v:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65))", CONVERTER); BOOST_CHECK(ms_dup1); @@ -290,6 +296,15 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // Same when the duplicates are on different levels in the tree const auto ms_dup4 = miniscript::FromString("thresh(2,pkh(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),s:pk(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),a:and_b(dv:older(1),s:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)))", CONVERTER); BOOST_CHECK(ms_dup4 && !ms_dup4->IsSane() && !ms_dup4->CheckDuplicateKey()); + // Test we find the first insane sub closer to be a leaf node. This fragment is insane for two reasons: + // 1. It can be spent without a signature + // 2. It contains timelock mixes + // We'll report the timelock mix error, as it's "deeper" (closer to be a leaf node) than the "no 's' property" + // error is. + const auto ms_ins = miniscript::FromString("or_i(and_b(after(1),a:after(1000000000)),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204))", CONVERTER); + BOOST_CHECK(ms_ins && ms_ins->IsValid() && !ms_ins->IsSane()); + const auto insane_sub = ms_ins->FindInsaneSub(); + BOOST_CHECK(insane_sub && *insane_sub->ToString(CONVERTER) == "and_b(after(1),a:after(1000000000))"); // Timelock tests Test("after(100)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only heightlock diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 115c4b9b24..f6642d3218 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -4,7 +4,7 @@ #include <chainparams.h> #include <clientversion.h> -#include <compat.h> +#include <compat/compat.h> #include <cstdint> #include <net.h> #include <net_processing.h> @@ -58,7 +58,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) std::string pszDest; std::unique_ptr<CNode> pnode1 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -77,7 +76,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode1->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode2 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/1, @@ -96,7 +94,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode2->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode3 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -115,7 +112,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode3->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode4 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/1, @@ -629,7 +625,6 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); std::unique_ptr<CNode> pnode = std::make_unique<CNode>(/*id=*/0, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -648,7 +643,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) pnode->SetAddrLocal(addrLocal); // before patch, this causes undefined behavior detectable with clang's -fsanitize=memory - GetLocalAddrForPeer(&*pnode); + GetLocalAddrForPeer(*pnode); // suppress no-checks-run warning; if this test fails, it's by triggering a sanitizer BOOST_CHECK(1); @@ -675,19 +670,15 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) const uint16_t bind_port = 20001; m_node.args->ForceSetArg("-bind", strprintf("3.4.5.6:%u", bind_port)); - const uint32_t current_time = static_cast<uint32_t>(GetAdjustedTime()); - SetMockTime(current_time); - // Our address:port as seen from the peer, completely different from the above. in_addr peer_us_addr; peer_us_addr.s_addr = htonl(0x02030405); - const CAddress peer_us{CService{peer_us_addr, 20002}, NODE_NETWORK, current_time}; + const CService peer_us{peer_us_addr, 20002}; // Create a peer with a routable IPv4 address (outbound). in_addr peer_out_in_addr; peer_out_in_addr.s_addr = htonl(0x01020304); CNode peer_out{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_out_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -700,16 +691,15 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) peer_out.SetAddrLocal(peer_us); // Without the fix peer_us:8333 is chosen instead of the proper peer_us:bind_port. - auto chosen_local_addr = GetLocalAddrForPeer(&peer_out); + auto chosen_local_addr = GetLocalAddrForPeer(peer_out); BOOST_REQUIRE(chosen_local_addr); - const CAddress expected{CService{peer_us_addr, bind_port}, NODE_NETWORK, current_time}; + const CService expected{peer_us_addr, bind_port}; BOOST_CHECK(*chosen_local_addr == expected); // Create a peer with a routable IPv4 address (inbound). in_addr peer_in_in_addr; peer_in_in_addr.s_addr = htonl(0x05060708); CNode peer_in{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_in_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -722,7 +712,7 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) peer_in.SetAddrLocal(peer_us); // Without the fix peer_us:8333 is chosen instead of the proper peer_us:peer_us.GetPort(). - chosen_local_addr = GetLocalAddrForPeer(&peer_in); + chosen_local_addr = GetLocalAddrForPeer(peer_in); BOOST_REQUIRE(chosen_local_addr); BOOST_CHECK(*chosen_local_addr == peer_us); @@ -837,7 +827,6 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) in_addr peer_in_addr; peer_in_addr.s_addr = htonl(0x01020304); CNode peer{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -857,7 +846,7 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) *static_cast<TestChainState*>(&m_node.chainman->ActiveChainstate()); chainstate.JumpOutOfIbd(); - m_node.peerman->InitializeNode(&peer); + m_node.peerman->InitializeNode(peer, NODE_NETWORK); std::atomic<bool> interrupt_dummy{false}; std::chrono::microseconds time_received_dummy{0}; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 224dc88d0f..c2d2fa37b4 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -480,21 +480,21 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) // try a few edge cases for port, service flags and time. static const std::vector<CAddress> fixture_addresses({ - CAddress( + CAddress{ CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0 /* port */), NODE_NONE, - 0x4966bc61U /* Fri Jan 9 02:54:25 UTC 2009 */ - ), - CAddress( + NodeSeconds{0x4966bc61s}, /* Fri Jan 9 02:54:25 UTC 2009 */ + }, + CAddress{ CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0x00f1 /* port */), NODE_NETWORK, - 0x83766279U /* Tue Nov 22 11:22:33 UTC 2039 */ - ), - CAddress( + NodeSeconds{0x83766279s}, /* Tue Nov 22 11:22:33 UTC 2039 */ + }, + CAddress{ CService(CNetAddr(in6_addr(IN6ADDR_LOOPBACK_INIT)), 0xf1f2 /* port */), static_cast<ServiceFlags>(NODE_WITNESS | NODE_COMPACT_FILTERS | NODE_NETWORK_LIMITED), - 0xffffffffU /* Sun Feb 7 06:28:15 UTC 2106 */ - ) + NodeSeconds{0xffffffffs}, /* Sun Feb 7 06:28:15 UTC 2106 */ + }, }); // fixture_addresses should equal to this when serialized in V1 format. diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 9b2760fd1c..96fb28dc9f 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -53,6 +53,16 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); BOOST_CHECK(ctx1.rand256() == ctx2.rand256()); BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); + { + struct MicroClock { + using duration = std::chrono::microseconds; + }; + FastRandomContext ctx{true}; + // Check with clock type + BOOST_CHECK_EQUAL(47222, ctx.rand_uniform_duration<MicroClock>(1s).count()); + // Check with time-point type + BOOST_CHECK_EQUAL(2782, ctx.rand_uniform_duration<SteadySeconds>(9h).count()); + } // Check that a nondeterministic ones are not g_mock_deterministic_tests = false; diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp new file mode 100644 index 0000000000..e597081afd --- /dev/null +++ b/src/test/rbf_tests.cpp @@ -0,0 +1,230 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <policy/rbf.h> +#include <random.h> +#include <txmempool.h> +#include <util/system.h> +#include <util/time.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> +#include <optional> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(rbf_tests, TestingSetup) + +static inline CTransactionRef make_tx(const std::vector<CTransactionRef>& inputs, + const std::vector<CAmount>& output_values) +{ + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(inputs.size()); + tx.vout.resize(output_values.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + tx.vin[i].prevout.hash = inputs[i]->GetHash(); + tx.vin[i].prevout.n = 0; + // Add a witness so wtxid != txid + CScriptWitness witness; + witness.stack.push_back(std::vector<unsigned char>(i + 10)); + tx.vin[i].scriptWitness = witness; + } + for (size_t i = 0; i < output_values.size(); ++i) { + tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx.vout[i].nValue = output_values[i]; + } + return MakeTransactionRef(tx); +} + +static void add_descendants(const CTransactionRef& tx, int32_t num_descendants, CTxMemPool& pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(pool.cs); + TestMemPoolEntryHelper entry; + // Assumes this isn't already spent in mempool + auto tx_to_spend = tx; + for (int32_t i{0}; i < num_descendants; ++i) { + auto next_tx = make_tx(/*inputs=*/{tx_to_spend}, /*output_values=*/{(50 - i) * CENT}); + pool.addUnchecked(entry.FromTx(next_tx)); + tx_to_spend = next_tx; + } +} + +BOOST_FIXTURE_TEST_CASE(rbf_helper_functions, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/100}; + const CAmount normal_fee{CENT/10}; + const CAmount high_fee{CENT}; + + // Create a parent tx1 and child tx2 with normal fees: + const auto tx1 = make_tx(/*inputs=*/ {m_coinbase_txns[0]}, /*output_values=*/ {10 * COIN}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + const auto tx2 = make_tx(/*inputs=*/ {tx1}, /*output_values=*/ {995 * CENT}); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + + // Create a low-feerate parent tx3 and high-feerate child tx4 (cpfp) + const auto tx3 = make_tx(/*inputs=*/ {m_coinbase_txns[1]}, /*output_values=*/ {1099 * CENT}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx3)); + const auto tx4 = make_tx(/*inputs=*/ {tx3}, /*output_values=*/ {999 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + + // Create a parent tx5 and child tx6 where both have very low fees + const auto tx5 = make_tx(/*inputs=*/ {m_coinbase_txns[2]}, /*output_values=*/ {1099 * CENT}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + const auto tx6 = make_tx(/*inputs=*/ {tx3}, /*output_values=*/ {1098 * CENT}); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + // Make tx6's modified fee much higher than its base fee. This should cause it to pass + // the fee-related checks despite being low-feerate. + pool.PrioritiseTransaction(tx6->GetHash(), 1 * COIN); + + // Two independent high-feerate transactions, tx7 and tx8 + const auto tx7 = make_tx(/*inputs=*/ {m_coinbase_txns[3]}, /*output_values=*/ {999 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx7)); + const auto tx8 = make_tx(/*inputs=*/ {m_coinbase_txns[4]}, /*output_values=*/ {999 * CENT}); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); + + const auto entry1 = pool.GetIter(tx1->GetHash()).value(); + const auto entry2 = pool.GetIter(tx2->GetHash()).value(); + const auto entry3 = pool.GetIter(tx3->GetHash()).value(); + const auto entry4 = pool.GetIter(tx4->GetHash()).value(); + const auto entry5 = pool.GetIter(tx5->GetHash()).value(); + const auto entry6 = pool.GetIter(tx6->GetHash()).value(); + const auto entry7 = pool.GetIter(tx7->GetHash()).value(); + const auto entry8 = pool.GetIter(tx8->GetHash()).value(); + + BOOST_CHECK_EQUAL(entry1->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry2->GetFee(), normal_fee); + BOOST_CHECK_EQUAL(entry3->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry4->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry5->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry6->GetFee(), low_fee); + BOOST_CHECK_EQUAL(entry7->GetFee(), high_fee); + BOOST_CHECK_EQUAL(entry8->GetFee(), high_fee); + + CTxMemPool::setEntries set_12_normal{entry1, entry2}; + CTxMemPool::setEntries set_34_cpfp{entry3, entry4}; + CTxMemPool::setEntries set_56_low{entry5, entry6}; + CTxMemPool::setEntries all_entries{entry1, entry2, entry3, entry4, entry5, entry6, entry7, entry8}; + CTxMemPool::setEntries empty_set; + + const auto unused_txid{GetRandHash()}; + + // Tests for PaysMoreThanConflicts + // These tests use feerate, not absolute fee. + BOOST_CHECK(PaysMoreThanConflicts(/*iters_conflicting=*/set_12_normal, + /*replacement_feerate=*/CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize() + 2), + /*txid=*/unused_txid).has_value()); + // Replacement must be strictly greater than the originals. + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee(), entry1->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts(set_12_normal, CFeeRate(entry1->GetModifiedFee() + 1, entry1->GetTxSize()), unused_txid) == std::nullopt); + // These tests use modified fees (including prioritisation), not base fees. + BOOST_CHECK(PaysMoreThanConflicts({entry5}, CFeeRate(entry5->GetModifiedFee() + 1, entry5->GetTxSize()), unused_txid) == std::nullopt); + BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetFee() + 1, entry6->GetTxSize()), unused_txid).has_value()); + BOOST_CHECK(PaysMoreThanConflicts({entry6}, CFeeRate(entry6->GetModifiedFee() + 1, entry6->GetTxSize()), unused_txid) == std::nullopt); + // PaysMoreThanConflicts checks individual feerate, not ancestor feerate. This test compares + // replacement_feerate and entry4's feerate, which are the same. The replacement_feerate is + // considered too low even though entry4 has a low ancestor feerate. + BOOST_CHECK(PaysMoreThanConflicts(set_34_cpfp, CFeeRate(entry4->GetModifiedFee(), entry4->GetTxSize()), unused_txid).has_value()); + + // Tests for EntriesAndTxidsDisjoint + BOOST_CHECK(EntriesAndTxidsDisjoint(empty_set, {tx1->GetHash()}, unused_txid) == std::nullopt); + BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx3->GetHash()}, unused_txid) == std::nullopt); + // EntriesAndTxidsDisjoint uses txids, not wtxids. + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx2->GetWitnessHash()}, unused_txid) == std::nullopt); + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx2->GetHash()}, unused_txid).has_value()); + BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx1->GetHash()}, unused_txid).has_value()); + BOOST_CHECK(EntriesAndTxidsDisjoint(set_12_normal, {tx2->GetHash()}, unused_txid).has_value()); + // EntriesAndTxidsDisjoint does not calculate descendants of iters_conflicting; it uses whatever + // the caller passed in. As such, no error is returned even though entry2 is a descendant of tx1. + BOOST_CHECK(EntriesAndTxidsDisjoint({entry2}, {tx1->GetHash()}, unused_txid) == std::nullopt); + + // Tests for PaysForRBF + const CFeeRate incremental_relay_feerate{DEFAULT_INCREMENTAL_RELAY_FEE}; + const CFeeRate higher_relay_feerate{2 * DEFAULT_INCREMENTAL_RELAY_FEE}; + // Must pay at least as much as the original. + BOOST_CHECK(PaysForRBF(/*original_fees=*/high_fee, + /*replacement_fees=*/high_fee, + /*replacement_vsize=*/1, + /*relay_fee=*/CFeeRate(0), + /*txid=*/unused_txid) + == std::nullopt); + BOOST_CHECK(PaysForRBF(high_fee, high_fee - 1, 1, CFeeRate(0), unused_txid).has_value()); + BOOST_CHECK(PaysForRBF(high_fee + 1, high_fee, 1, CFeeRate(0), unused_txid).has_value()); + // Additional fees must cover the replacement's vsize at incremental relay fee + BOOST_CHECK(PaysForRBF(high_fee, high_fee + 1, 2, incremental_relay_feerate, unused_txid).has_value()); + BOOST_CHECK(PaysForRBF(high_fee, high_fee + 2, 2, incremental_relay_feerate, unused_txid) == std::nullopt); + BOOST_CHECK(PaysForRBF(high_fee, high_fee + 2, 2, higher_relay_feerate, unused_txid).has_value()); + BOOST_CHECK(PaysForRBF(high_fee, high_fee + 4, 2, higher_relay_feerate, unused_txid) == std::nullopt); + BOOST_CHECK(PaysForRBF(low_fee, high_fee, 99999999, incremental_relay_feerate, unused_txid).has_value()); + BOOST_CHECK(PaysForRBF(low_fee, high_fee + 99999999, 99999999, incremental_relay_feerate, unused_txid) == std::nullopt); + + // Tests for GetEntriesForConflicts + CTxMemPool::setEntries all_parents{entry1, entry3, entry5, entry7, entry8}; + CTxMemPool::setEntries all_children{entry2, entry4, entry6}; + const std::vector<CTransactionRef> parent_inputs({m_coinbase_txns[0], m_coinbase_txns[1], m_coinbase_txns[2], + m_coinbase_txns[3], m_coinbase_txns[4]}); + const auto conflicts_with_parents = make_tx(parent_inputs, {50 * CENT}); + CTxMemPool::setEntries all_conflicts; + BOOST_CHECK(GetEntriesForConflicts(/*tx=*/ *conflicts_with_parents.get(), + /*pool=*/ pool, + /*iters_conflicting=*/ all_parents, + /*all_conflicts=*/ all_conflicts) == std::nullopt); + BOOST_CHECK(all_conflicts == all_entries); + auto conflicts_size = all_conflicts.size(); + all_conflicts.clear(); + + add_descendants(tx2, 23, pool); + BOOST_CHECK(GetEntriesForConflicts(*conflicts_with_parents.get(), pool, all_parents, all_conflicts) == std::nullopt); + conflicts_size += 23; + BOOST_CHECK_EQUAL(all_conflicts.size(), conflicts_size); + all_conflicts.clear(); + + add_descendants(tx4, 23, pool); + BOOST_CHECK(GetEntriesForConflicts(*conflicts_with_parents.get(), pool, all_parents, all_conflicts) == std::nullopt); + conflicts_size += 23; + BOOST_CHECK_EQUAL(all_conflicts.size(), conflicts_size); + all_conflicts.clear(); + + add_descendants(tx6, 23, pool); + BOOST_CHECK(GetEntriesForConflicts(*conflicts_with_parents.get(), pool, all_parents, all_conflicts) == std::nullopt); + conflicts_size += 23; + BOOST_CHECK_EQUAL(all_conflicts.size(), conflicts_size); + all_conflicts.clear(); + + add_descendants(tx7, 23, pool); + BOOST_CHECK(GetEntriesForConflicts(*conflicts_with_parents.get(), pool, all_parents, all_conflicts) == std::nullopt); + conflicts_size += 23; + BOOST_CHECK_EQUAL(all_conflicts.size(), conflicts_size); + BOOST_CHECK_EQUAL(all_conflicts.size(), 100); + all_conflicts.clear(); + + // Exceeds maximum number of conflicts. + add_descendants(tx8, 1, pool); + BOOST_CHECK(GetEntriesForConflicts(*conflicts_with_parents.get(), pool, all_parents, all_conflicts).has_value()); + + // Tests for HasNoNewUnconfirmed + const auto spends_unconfirmed = make_tx({tx1}, {36 * CENT}); + for (const auto& input : spends_unconfirmed->vin) { + // Spends unconfirmed inputs. + BOOST_CHECK(pool.exists(GenTxid::Txid(input.prevout.hash))); + } + BOOST_CHECK(HasNoNewUnconfirmed(/*tx=*/ *spends_unconfirmed.get(), + /*pool=*/ pool, + /*iters_conflicting=*/ all_entries) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, {entry2}) == std::nullopt); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_unconfirmed.get(), pool, empty_set).has_value()); + + const auto spends_new_unconfirmed = make_tx({tx1, tx8}, {36 * CENT}); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, {entry2}).has_value()); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_new_unconfirmed.get(), pool, all_entries).has_value()); + + const auto spends_conflicting_confirmed = make_tx({m_coinbase_txns[0], m_coinbase_txns[1]}, {45 * CENT}); + BOOST_CHECK(HasNoNewUnconfirmed(*spends_conflicting_confirmed.get(), pool, {entry1, entry3}) == std::nullopt); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 05bb89ab55..9e7a376d6b 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1514,8 +1514,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) CScriptWitness wit; scriptPubKey << OP_1; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << spendTx; @@ -1537,8 +1537,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) CScriptWitness wit; scriptPubKey << OP_EQUAL; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << spendTx; @@ -1560,8 +1560,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) CScriptWitness wit; scriptPubKey << OP_EQUAL; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << spendTx; @@ -1583,8 +1583,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) CScriptWitness wit; scriptPubKey << OP_EQUAL; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << 0xffffffff; @@ -1606,8 +1606,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) CScriptWitness wit; scriptPubKey << OP_EQUAL; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << spendTx; @@ -1629,8 +1629,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) CScriptWitness wit; scriptPubKey << OP_EQUAL; - CTransaction creditTx = BuildCreditingTransaction(scriptPubKey, 1); - CTransaction spendTx = BuildSpendingTransaction(scriptSig, wit, creditTx); + CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; + CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << spendTx; @@ -1813,7 +1813,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) BOOST_CHECK_EQUAL(HexStr(sighash), input["intermediary"]["sigHash"].get_str()); // To verify the sigmsg, hash the expected sigmsg, and compare it with the (expected) sighash. - BOOST_CHECK_EQUAL(HexStr((CHashWriter(HASHER_TAPSIGHASH) << Span{ParseHex(input["intermediary"]["sigMsg"].get_str())}).GetSHA256()), input["intermediary"]["sigHash"].get_str()); + BOOST_CHECK_EQUAL(HexStr((HashWriter{HASHER_TAPSIGHASH} << Span{ParseHex(input["intermediary"]["sigMsg"].get_str())}).GetSHA256()), input["intermediary"]["sigHash"].get_str()); } } diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index 01a402833d..8376ec1a68 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <compat.h> +#include <compat/compat.h> #include <test/util/setup_common.h> #include <threadinterrupt.h> #include <util/sock.h> diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 3f5353b5a2..f160bb08a5 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -7,11 +7,6 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#if defined(WIN32) && !defined(__kernel_entry) -// A workaround for boost 1.71 incompatibility with mingw-w64 compiler. -// For details see https://github.com/bitcoin/bitcoin/pull/22348. -#define __kernel_entry -#endif #if defined(__GNUC__) // Boost 1.78 requires the following workaround. // See: https://github.com/boostorg/process/issues/235 diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 15213f826b..62c7ddb673 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <index/txindex.h> +#include <interfaces/chain.h> #include <script/standard.h> #include <test/util/setup_common.h> #include <util/time.h> @@ -15,7 +16,7 @@ BOOST_AUTO_TEST_SUITE(txindex_tests) BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) { - TxIndex txindex(1 << 20, true); + TxIndex txindex(interfaces::MakeChain(m_node), 1 << 20, true); CTransactionRef tx_disk; uint256 block_hash; @@ -28,7 +29,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // BlockUntilSyncedToCurrentChain should return false before txindex is started. BOOST_CHECK(!txindex.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(txindex.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(txindex.Start()); // Allow tx index to catch up with the block index. constexpr int64_t timeout_ms = 10 * 1000; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 5ac504c24f..13e0e684b8 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -16,7 +16,7 @@ #include <boost/test/unit_test.hpp> -const auto NoMalleation = [](CAutoFile& file, node::SnapshotMetadata& meta){}; +const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to @@ -32,7 +32,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); fs::path snapshot_path = root / fs::u8path(tfm::format("test_snapshot.%d.dat", height)); FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_outfile{outfile}; UniValue result = CreateUTXOSnapshot( node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); @@ -42,7 +42,7 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma // Read the written snapshot in and then activate it. // FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + AutoFile auto_infile{infile}; node::SnapshotMetadata metadata; auto_infile >> metadata; diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index e70ad036bd..223db16c6c 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -16,6 +16,7 @@ void ConnmanTestMsg::Handshake(CNode& node, bool successfully_connected, ServiceFlags remote_services, + ServiceFlags local_services, NetPermissionFlags permission_flags, int32_t version, bool relay_txs) @@ -24,7 +25,7 @@ void ConnmanTestMsg::Handshake(CNode& node, auto& connman{*this}; const CNetMsgMaker mm{0}; - peerman.InitializeNode(&node); + peerman.InitializeNode(node, local_services); CSerializedNetMsg msg_version{ mm.Make(NetMsgType::VERSION, @@ -51,10 +52,10 @@ void ConnmanTestMsg::Handshake(CNode& node, if (node.fDisconnect) return; assert(node.nVersion == version); assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); - assert(node.nServices == remote_services); CNodeStateStats statestats; assert(peerman.GetNodeStateStats(node.GetId(), statestats)); assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); + assert(statestats.their_services == remote_services); node.m_permissionFlags = permission_flags; if (successfully_connected) { CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; diff --git a/src/test/util/net.h b/src/test/util/net.h index 0cf55f8a22..ec6b4e6e88 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_TEST_UTIL_NET_H #define BITCOIN_TEST_UTIL_NET_H -#include <compat.h> +#include <compat/compat.h> #include <node/eviction.h> #include <netaddress.h> #include <net.h> @@ -42,6 +42,7 @@ struct ConnmanTestMsg : public CConnman { void Handshake(CNode& node, bool successfully_connected, ServiceFlags remote_services, + ServiceFlags local_services, NetPermissionFlags permission_flags, int32_t version, bool relay_txs); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index de53499088..67984721a3 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -54,8 +54,6 @@ using node::BlockAssembler; using node::CalculateCacheSizes; -using node::fPruneMode; -using node::fReindex; using node::LoadChainstate; using node::NodeContext; using node::RegenerateCommitments; @@ -183,8 +181,8 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_cache_sizes = CalculateCacheSizes(m_args); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(m_cache_sizes.block_tree_db, true); @@ -218,25 +216,20 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - auto maybe_load_error = LoadChainstate(fReindex.load(), - *Assert(m_node.chainman.get()), - Assert(m_node.mempool.get()), - fPruneMode, - m_args.GetBoolArg("-reindex-chainstate", false), - m_cache_sizes.block_tree_db, - m_cache_sizes.coins_db, - m_cache_sizes.coins, - /*block_tree_db_in_memory=*/true, - /*coins_db_in_memory=*/true); - assert(!maybe_load_error.has_value()); - - auto maybe_verify_error = VerifyLoadedChainstate( - *Assert(m_node.chainman), - fReindex.load(), - m_args.GetBoolArg("-reindex-chainstate", false), - m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS), - m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL)); - assert(!maybe_verify_error.has_value()); + node::ChainstateLoadOptions options; + options.mempool = Assert(m_node.mempool.get()); + options.block_tree_db_in_memory = true; + options.coins_db_in_memory = true; + options.reindex = node::fReindex; + options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); + options.prune = node::fPruneMode; + options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + auto [status, error] = LoadChainstate(*Assert(m_node.chainman), m_cache_sizes, options); + assert(status == node::ChainstateLoadStatus::SUCCESS); + + std::tie(status, error) = VerifyLoadedChainstate(*Assert(m_node.chainman), options); + assert(status == node::ChainstateLoadStatus::SUCCESS); BlockValidationState state; if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 102de74389..78adb521ac 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -24,8 +24,8 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { const ChainstateManager::Options chainman_opts{ - Params(), - GetAdjustedTime, + .chainparams = Params(), + .adjusted_time_callback = GetAdjustedTime, }; ChainstateManager manager{chainman_opts}; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 6dc522b421..14de96ff41 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -193,7 +193,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) // Should not load malleated snapshots BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // A UTXO is missing but count is correct metadata.m_coins_count -= 1; @@ -204,22 +204,22 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) auto_infile >> coin; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is larger than coins in file metadata.m_coins_count += 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Coins count is smaller than coins in file metadata.m_coins_count -= 1; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ZERO; })); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { // Wrong hash metadata.m_base_blockhash = uint256::ONE; })); diff --git a/src/threadinterrupt.cpp b/src/threadinterrupt.cpp index 340106ed99..e28b447c1d 100644 --- a/src/threadinterrupt.cpp +++ b/src/threadinterrupt.cpp @@ -28,18 +28,8 @@ void CThreadInterrupt::operator()() cond.notify_all(); } -bool CThreadInterrupt::sleep_for(std::chrono::milliseconds rel_time) +bool CThreadInterrupt::sleep_for(Clock::duration rel_time) { WAIT_LOCK(mut, lock); return !cond.wait_for(lock, rel_time, [this]() { return flag.load(std::memory_order_acquire); }); } - -bool CThreadInterrupt::sleep_for(std::chrono::seconds rel_time) -{ - return sleep_for(std::chrono::duration_cast<std::chrono::milliseconds>(rel_time)); -} - -bool CThreadInterrupt::sleep_for(std::chrono::minutes rel_time) -{ - return sleep_for(std::chrono::duration_cast<std::chrono::milliseconds>(rel_time)); -} diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index 992016b4f6..363aab39ce 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -19,13 +19,12 @@ class CThreadInterrupt { public: + using Clock = std::chrono::steady_clock; CThreadInterrupt(); explicit operator bool() const; void operator()() EXCLUSIVE_LOCKS_REQUIRED(!mut); void reset(); - bool sleep_for(std::chrono::milliseconds rel_time) EXCLUSIVE_LOCKS_REQUIRED(!mut); - bool sleep_for(std::chrono::seconds rel_time) EXCLUSIVE_LOCKS_REQUIRED(!mut); - bool sleep_for(std::chrono::minutes rel_time) EXCLUSIVE_LOCKS_REQUIRED(!mut); + bool sleep_for(Clock::duration rel_time) EXCLUSIVE_LOCKS_REQUIRED(!mut); private: std::condition_variable cond; diff --git a/src/timedata.h b/src/timedata.h index 2f039d5465..ed2d8639f7 100644 --- a/src/timedata.h +++ b/src/timedata.h @@ -5,9 +5,12 @@ #ifndef BITCOIN_TIMEDATA_H #define BITCOIN_TIMEDATA_H +#include <util/time.h> + #include <algorithm> -#include <assert.h> -#include <stdint.h> +#include <cassert> +#include <chrono> +#include <cstdint> #include <vector> static const int64_t DEFAULT_MAX_TIME_ADJUSTMENT = 70 * 60; @@ -73,6 +76,7 @@ public: /** Functions to keep track of adjusted P2P time */ int64_t GetTimeOffset(); int64_t GetAdjustedTime(); +inline NodeSeconds AdjustedTime() { return Now<NodeSeconds>() + std::chrono::seconds{GetTimeOffset()}; } void AddTimeData(const CNetAddr& ip, int64_t nTime); /** diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index d6e792a55f..3a21a79a34 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -7,7 +7,7 @@ #include <chainparams.h> #include <chainparamsbase.h> -#include <compat.h> +#include <compat/compat.h> #include <crypto/hmac_sha256.h> #include <net.h> #include <netaddress.h> diff --git a/src/txdb.cpp b/src/txdb.cpp index c048c2d92a..bad3bb80a9 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -310,7 +310,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object - CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash()); + CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash()); pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; diff --git a/src/txdb.h b/src/txdb.h index faa543b412..a04596f7bb 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,6 +8,7 @@ #include <coins.h> #include <dbwrapper.h> +#include <sync.h> #include <memory> #include <optional> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index aeaa10034e..7eff6bdbe3 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1210,14 +1210,14 @@ void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, } } -bool CTxMemPool::IsLoaded() const +bool CTxMemPool::GetLoadTried() const { LOCK(cs); - return m_is_loaded; + return m_load_tried; } -void CTxMemPool::SetIsLoaded(bool loaded) +void CTxMemPool::SetLoadTried(bool load_tried) { LOCK(cs); - m_is_loaded = loaded; + m_load_tried = load_tried; } diff --git a/src/txmempool.h b/src/txmempool.h index 6e37f59f2e..d7d308038c 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -451,7 +451,7 @@ protected: void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool m_is_loaded GUARDED_BY(cs){false}; + bool m_load_tried GUARDED_BY(cs){false}; CFeeRate GetMinFee(size_t sizelimit) const; @@ -728,11 +728,17 @@ public: */ void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; - /** @returns true if the mempool is fully loaded */ - bool IsLoaded() const; + /** + * @returns true if we've made an attempt to load the mempool regardless of + * whether the attempt was successful or not + */ + bool GetLoadTried() const; - /** Sets the current loaded state */ - void SetIsLoaded(bool loaded); + /** + * Set whether or not we've made an attempt to load the mempool (regardless + * of whether the attempt was successful or not) + */ + void SetLoadTried(bool load_tried); unsigned long size() const { diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index ed4783f1a5..69ae8ea582 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -102,7 +102,7 @@ void TxOrphanage::EraseForPeer(NodeId peer) if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer); } -unsigned int TxOrphanage::LimitOrphans(unsigned int max_orphans) +void TxOrphanage::LimitOrphans(unsigned int max_orphans) { AssertLockHeld(g_cs_orphans); @@ -135,7 +135,7 @@ unsigned int TxOrphanage::LimitOrphans(unsigned int max_orphans) EraseTx(m_orphan_list[randompos]->first); ++nEvicted; } - return nEvicted; + if (nEvicted > 0) LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); } void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx, std::set<uint256>& orphan_work_set) const diff --git a/src/txorphanage.h b/src/txorphanage.h index 24c8318f36..9363e6f733 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -41,7 +41,7 @@ public: void EraseForBlock(const CBlock& block) LOCKS_EXCLUDED(::g_cs_orphans); /** Limit the orphanage to the given maximum */ - unsigned int LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + void LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); /** Add any orphans that list a particular tx as a parent into a peer's work set * (ie orphans that may have found their final missing parent, and so should be reconsidered for the mempool) */ diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 7f9a6aaffd..230a5fd096 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -24,27 +24,25 @@ public: typ = initialType; val = initialStr; } - UniValue(uint64_t val_) { - setInt(val_); - } - UniValue(int64_t val_) { - setInt(val_); - } - UniValue(bool val_) { - setBool(val_); - } - UniValue(int val_) { - setInt(val_); - } - UniValue(double val_) { - setFloat(val_); - } - UniValue(const std::string& val_) { - setStr(val_); - } - UniValue(const char *val_) { - std::string s(val_); - setStr(s); + template <typename Ref, typename T = std::remove_cv_t<std::remove_reference_t<Ref>>, + std::enable_if_t<std::is_floating_point_v<T> || // setFloat + std::is_same_v<bool, T> || // setBool + std::is_signed_v<T> || std::is_unsigned_v<T> || // setInt + std::is_constructible_v<std::string, T>, // setStr + bool> = true> + UniValue(Ref&& val) + { + if constexpr (std::is_floating_point_v<T>) { + setFloat(val); + } else if constexpr (std::is_same_v<bool, T>) { + setBool(val); + } else if constexpr (std::is_signed_v<T>) { + setInt(int64_t{val}); + } else if constexpr (std::is_unsigned_v<T>) { + setInt(uint64_t{val}); + } else { + setStr(std::string{std::forward<Ref>(val)}); + } } void clear(); @@ -82,14 +80,14 @@ public: bool isArray() const { return (typ == VARR); } bool isObject() const { return (typ == VOBJ); } - bool push_back(const UniValue& val); - bool push_backV(const std::vector<UniValue>& vec); + void push_back(const UniValue& val); + void push_backV(const std::vector<UniValue>& vec); template <class It> - bool push_backV(It first, It last); + void push_backV(It first, It last); void __pushKV(const std::string& key, const UniValue& val); - bool pushKV(const std::string& key, const UniValue& val); - bool pushKVs(const UniValue& obj); + void pushKV(const std::string& key, const UniValue& val); + void pushKVs(const UniValue& obj); std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; @@ -106,6 +104,7 @@ private: std::vector<std::string> keys; std::vector<UniValue> values; + void checkType(const VType& expected) const; bool findKey(const std::string& key, size_t& retIdx) const; void writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; void writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; @@ -116,19 +115,7 @@ public: const std::vector<std::string>& getKeys() const; const std::vector<UniValue>& getValues() const; template <typename Int> - auto getInt() const - { - static_assert(std::is_integral<Int>::value); - if (typ != VNUM) { - throw std::runtime_error("JSON value is not an integer as expected"); - } - Int result; - const auto [first_nonmatching, error_condition] = std::from_chars(val.data(), val.data() + val.size(), result); - if (first_nonmatching != val.data() + val.size() || error_condition != std::errc{}) { - throw std::runtime_error("JSON integer out of range"); - } - return result; - } + Int getInt() const; bool get_bool() const; const std::string& get_str() const; double get_real() const; @@ -140,11 +127,23 @@ public: }; template <class It> -bool UniValue::push_backV(It first, It last) +void UniValue::push_backV(It first, It last) { - if (typ != VARR) return false; + checkType(VARR); values.insert(values.end(), first, last); - return true; +} + +template <typename Int> +Int UniValue::getInt() const +{ + static_assert(std::is_integral<Int>::value); + checkType(VNUM); + Int result; + const auto [first_nonmatching, error_condition] = std::from_chars(val.data(), val.data() + val.size(), result); + if (first_nonmatching != val.data() + val.size() || error_condition != std::errc{}) { + throw std::runtime_error("JSON integer out of range"); + } + return result; } enum jtokentype { diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 3553995c28..051bc8eba6 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -108,53 +108,46 @@ bool UniValue::setObject() return true; } -bool UniValue::push_back(const UniValue& val_) +void UniValue::push_back(const UniValue& val_) { - if (typ != VARR) - return false; + checkType(VARR); values.push_back(val_); - return true; } -bool UniValue::push_backV(const std::vector<UniValue>& vec) +void UniValue::push_backV(const std::vector<UniValue>& vec) { - if (typ != VARR) - return false; + checkType(VARR); values.insert(values.end(), vec.begin(), vec.end()); - - return true; } void UniValue::__pushKV(const std::string& key, const UniValue& val_) { + checkType(VOBJ); + keys.push_back(key); values.push_back(val_); } -bool UniValue::pushKV(const std::string& key, const UniValue& val_) +void UniValue::pushKV(const std::string& key, const UniValue& val_) { - if (typ != VOBJ) - return false; + checkType(VOBJ); size_t idx; if (findKey(key, idx)) values[idx] = val_; else __pushKV(key, val_); - return true; } -bool UniValue::pushKVs(const UniValue& obj) +void UniValue::pushKVs(const UniValue& obj) { - if (typ != VOBJ || obj.typ != VOBJ) - return false; + checkType(VOBJ); + obj.checkType(VOBJ); for (size_t i = 0; i < obj.keys.size(); i++) __pushKV(obj.keys[i], obj.values.at(i)); - - return true; } void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const @@ -221,6 +214,14 @@ const UniValue& UniValue::operator[](size_t index) const return values.at(index); } +void UniValue::checkType(const VType& expected) const +{ + if (typ != expected) { + throw std::runtime_error{"JSON value of type " + std::string{uvTypeName(typ)} + " is not of expected type " + + std::string{uvTypeName(expected)}}; + } +} + const char *uvTypeName(UniValue::VType t) { switch (t) { diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index 9bbdb1fe69..5c58f388dd 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -46,8 +46,7 @@ bool ParseDouble(const std::string& str, double *out) const std::vector<std::string>& UniValue::getKeys() const { - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); + checkType(VOBJ); return keys; } @@ -60,22 +59,19 @@ const std::vector<UniValue>& UniValue::getValues() const bool UniValue::get_bool() const { - if (typ != VBOOL) - throw std::runtime_error("JSON value is not a boolean as expected"); + checkType(VBOOL); return getBool(); } const std::string& UniValue::get_str() const { - if (typ != VSTR) - throw std::runtime_error("JSON value is not a string as expected"); + checkType(VSTR); return getValStr(); } double UniValue::get_real() const { - if (typ != VNUM) - throw std::runtime_error("JSON value is not a number as expected"); + checkType(VNUM); double retval; if (!ParseDouble(getValStr(), &retval)) throw std::runtime_error("JSON double out of range"); @@ -84,15 +80,12 @@ double UniValue::get_real() const const UniValue& UniValue::get_obj() const { - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); + checkType(VOBJ); return *this; } const UniValue& UniValue::get_array() const { - if (typ != VARR) - throw std::runtime_error("JSON value is not an array as expected"); + checkType(VARR); return *this; } - diff --git a/src/univalue/sources.mk b/src/univalue/sources.mk index e156216378..5e4d9c3831 100644 --- a/src/univalue/sources.mk +++ b/src/univalue/sources.mk @@ -1,12 +1,8 @@ -# - All variables are namespaced with UNIVALUE_ to avoid colliding with -# downstream makefiles. # - All Variables ending in _HEADERS or _SOURCES confuse automake, so the # _INT postfix is applied. # - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used # as they interfere with automatic dependency generation -# - The %reldir% is the relative path from the Makefile.am. This allows -# downstreams to use these variables without having to manually account for -# the path change. +# - The %reldir% is the relative path from the Makefile.am. UNIVALUE_INCLUDE_DIR_INT = %reldir%/include @@ -29,9 +25,6 @@ UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp UNIVALUE_TEST_JSON_INT = UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp -UNIVALUE_TEST_NO_NUL_INT = -UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp - UNIVALUE_TEST_OBJECT_INT = UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp diff --git a/src/univalue/test/.gitignore b/src/univalue/test/.gitignore index 7b27cf0da2..5812c96b14 100644 --- a/src/univalue/test/.gitignore +++ b/src/univalue/test/.gitignore @@ -2,7 +2,6 @@ object unitester test_json -no_nul *.trs *.log diff --git a/src/univalue/test/no_nul.cpp b/src/univalue/test/no_nul.cpp deleted file mode 100644 index 3a7a727abb..0000000000 --- a/src/univalue/test/no_nul.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include <univalue.h> - -int main (int argc, char *argv[]) -{ - char buf[] = "___[1,2,3]___"; - UniValue val; - return val.read(buf + 3, 7) ? 0 : 1; -} diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 8a35bf914d..cf8c29ec67 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -13,9 +13,6 @@ #include <string> #include <vector> -#define BOOST_FIXTURE_TEST_SUITE(a, b) -#define BOOST_AUTO_TEST_CASE(funcName) void funcName() -#define BOOST_AUTO_TEST_SUITE_END() #define BOOST_CHECK(expr) assert(expr) #define BOOST_CHECK_EQUAL(v1, v2) assert((v1) == (v2)) #define BOOST_CHECK_THROW(stmt, excMatch) { \ @@ -35,9 +32,7 @@ } \ } -BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(univalue_constructor) +void univalue_constructor() { UniValue v1; BOOST_CHECK(v1.isNull()); @@ -85,7 +80,17 @@ BOOST_AUTO_TEST_CASE(univalue_constructor) BOOST_CHECK_EQUAL(v9.getValStr(), "zappa"); } -BOOST_AUTO_TEST_CASE(univalue_typecheck) +void univalue_push_throw() +{ + UniValue j; + BOOST_CHECK_THROW(j.push_back(1), std::runtime_error); + BOOST_CHECK_THROW(j.push_backV({1}), std::runtime_error); + BOOST_CHECK_THROW(j.__pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKVs({}), std::runtime_error); +} + +void univalue_typecheck() { UniValue v1; BOOST_CHECK(v1.setNumStr("1")); @@ -134,7 +139,7 @@ BOOST_AUTO_TEST_CASE(univalue_typecheck) BOOST_CHECK_THROW(vals[1].get_bool(), std::runtime_error); } -BOOST_AUTO_TEST_CASE(univalue_set) +void univalue_set() { UniValue v(UniValue::VSTR, "foo"); v.clear(); @@ -193,18 +198,18 @@ BOOST_AUTO_TEST_CASE(univalue_set) BOOST_CHECK(v.isNull()); } -BOOST_AUTO_TEST_CASE(univalue_array) +void univalue_array() { UniValue arr(UniValue::VARR); UniValue v((int64_t)1023LL); - BOOST_CHECK(arr.push_back(v)); + arr.push_back(v); std::string vStr("zippy"); - BOOST_CHECK(arr.push_back(vStr)); + arr.push_back(vStr); const char *s = "pippy"; - BOOST_CHECK(arr.push_back(s)); + arr.push_back(s); std::vector<UniValue> vec; v.setStr("boing"); @@ -213,13 +218,13 @@ BOOST_AUTO_TEST_CASE(univalue_array) v.setStr("going"); vec.push_back(v); - BOOST_CHECK(arr.push_backV(vec)); + arr.push_backV(vec); - BOOST_CHECK(arr.push_back((uint64_t) 400ULL)); - BOOST_CHECK(arr.push_back((int64_t) -400LL)); - BOOST_CHECK(arr.push_back((int) -401)); - BOOST_CHECK(arr.push_back(-40.1)); - BOOST_CHECK(arr.push_back(true)); + arr.push_back(uint64_t{400ULL}); + arr.push_back(int64_t{-400LL}); + arr.push_back(int{-401}); + arr.push_back(-40.1); + arr.push_back(true); BOOST_CHECK_EQUAL(arr.empty(), false); BOOST_CHECK_EQUAL(arr.size(), 10); @@ -252,7 +257,7 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK_EQUAL(arr.size(), 0); } -BOOST_AUTO_TEST_CASE(univalue_object) +void univalue_object() { UniValue obj(UniValue::VOBJ); std::string strKey, strVal; @@ -260,39 +265,39 @@ BOOST_AUTO_TEST_CASE(univalue_object) strKey = "age"; v.setInt(100); - BOOST_CHECK(obj.pushKV(strKey, v)); + obj.pushKV(strKey, v); strKey = "first"; strVal = "John"; - BOOST_CHECK(obj.pushKV(strKey, strVal)); + obj.pushKV(strKey, strVal); strKey = "last"; - const char *cVal = "Smith"; - BOOST_CHECK(obj.pushKV(strKey, cVal)); + const char* cVal = "Smith"; + obj.pushKV(strKey, cVal); strKey = "distance"; - BOOST_CHECK(obj.pushKV(strKey, (int64_t) 25)); + obj.pushKV(strKey, int64_t{25}); strKey = "time"; - BOOST_CHECK(obj.pushKV(strKey, (uint64_t) 3600)); + obj.pushKV(strKey, uint64_t{3600}); strKey = "calories"; - BOOST_CHECK(obj.pushKV(strKey, (int) 12)); + obj.pushKV(strKey, int{12}); strKey = "temperature"; - BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); + obj.pushKV(strKey, double{90.012}); strKey = "moon"; - BOOST_CHECK(obj.pushKV(strKey, true)); + obj.pushKV(strKey, true); strKey = "spoon"; - BOOST_CHECK(obj.pushKV(strKey, false)); + obj.pushKV(strKey, false); UniValue obj2(UniValue::VOBJ); - BOOST_CHECK(obj2.pushKV("cat1", 9000)); - BOOST_CHECK(obj2.pushKV("cat2", 12345)); + obj2.pushKV("cat1", 9000); + obj2.pushKV("cat2", 12345); - BOOST_CHECK(obj.pushKVs(obj2)); + obj.pushKVs(obj2); BOOST_CHECK_EQUAL(obj.empty(), false); BOOST_CHECK_EQUAL(obj.size(), 11); @@ -371,7 +376,7 @@ BOOST_AUTO_TEST_CASE(univalue_object) static const char *json1 = "[1.10000000,{\"key1\":\"str\\u0000\",\"key2\":800,\"key3\":{\"name\":\"martian http://test.com\"}}]"; -BOOST_AUTO_TEST_CASE(univalue_readwrite) +void univalue_readwrite() { UniValue v; BOOST_CHECK(v.read(json1)); @@ -414,11 +419,10 @@ BOOST_AUTO_TEST_CASE(univalue_readwrite) BOOST_CHECK(!v.read("{} 42")); } -BOOST_AUTO_TEST_SUITE_END() - int main (int argc, char *argv[]) { univalue_constructor(); + univalue_push_throw(); univalue_typecheck(); univalue_set(); univalue_array(); diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 94c149b39f..6344a5a0ab 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -149,6 +149,13 @@ void unescape_unicode_test() assert(val[0].get_str() == "\xf0\x9d\x85\xa1"); } +void no_nul_test() +{ + char buf[] = "___[1,2,3]___"; + UniValue val; + assert(val.read(buf + 3, 7)); +} + int main (int argc, char *argv[]) { for (const auto& f: filenames) { @@ -156,6 +163,7 @@ int main (int argc, char *argv[]) } unescape_unicode_test(); + no_nul_test(); return 0; } diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index b1db3b5f0d..33258d9962 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -198,7 +198,7 @@ std::vector<bool> DecodeAsmap(fs::path path) { std::vector<bool> bits; FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + AutoFile file{filestr}; if (file.IsNull()) { LogPrintf("Failed to open asmap file from disk\n"); return bits; diff --git a/src/util/message.cpp b/src/util/message.cpp index d395c4b0bc..028251a5a8 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -74,7 +74,7 @@ bool MessageSign( uint256 MessageHash(const std::string& message) { - CHashWriter hasher(SER_GETHASH, 0); + HashWriter hasher{}; hasher << MESSAGE_MAGIC << message; return hasher.GetHash(); diff --git a/src/util/result.h b/src/util/result.h index dcf5edaa5b..2f586a4c9b 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -6,6 +6,7 @@ #define BITCOIN_UTIL_RESULT_H #include <util/translation.h> + #include <variant> /* @@ -18,9 +19,9 @@ private: std::variant<bilingual_str, T> m_variant; public: - BResult() : m_variant(Untranslated("")) {} - BResult(const T& _obj) : m_variant(_obj) {} - BResult(const bilingual_str& error) : m_variant(error) {} + BResult() : m_variant{Untranslated("")} {} + BResult(T obj) : m_variant{std::move(obj)} {} + BResult(bilingual_str error) : m_variant{std::move(error)} {} /* Whether the function succeeded or not */ bool HasRes() const { return std::holds_alternative<T>(m_variant); } @@ -30,6 +31,11 @@ public: assert(HasRes()); return std::get<T>(m_variant); } + T ReleaseObj() + { + assert(HasRes()); + return std::move(std::get<T>(m_variant)); + } /* In case of failure, the error cause */ const bilingual_str& GetError() const { diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 2588575d81..125dbc7f18 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <compat.h> +#include <compat/compat.h> #include <logging.h> #include <threadinterrupt.h> #include <tinyformat.h> diff --git a/src/util/sock.h b/src/util/sock.h index b854609c22..38a7dc80d6 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_UTIL_SOCK_H #define BITCOIN_UTIL_SOCK_H -#include <compat.h> +#include <compat/compat.h> #include <threadinterrupt.h> #include <util/time.h> diff --git a/src/util/system.cpp b/src/util/system.cpp index f6f2828fc8..ce45fb2ed4 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -740,7 +740,7 @@ std::string ArgsManager::GetHelpMessage() const { const bool show_debug = GetBoolArg("-help-debug", false); - std::string usage = ""; + std::string usage; LOCK(cs_args); for (const auto& arg_map : m_available_args) { switch(arg_map.first) { diff --git a/src/util/system.h b/src/util/system.h index 04c66341d3..fa03e88920 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -14,7 +14,7 @@ #include <config/bitcoin-config.h> #endif -#include <compat.h> +#include <compat/compat.h> #include <compat/assumptions.h> #include <fs.h> #include <logging.h> diff --git a/src/util/time.cpp b/src/util/time.cpp index 2cafc55c69..f6d37347f8 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -7,7 +7,7 @@ #include <config/bitcoin-config.h> #endif -#include <compat.h> +#include <compat/compat.h> #include <tinyformat.h> #include <util/time.h> #include <util/check.h> diff --git a/src/util/time.h b/src/util/time.h index 9df69a953c..4f9bde5d56 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_UTIL_TIME_H #define BITCOIN_UTIL_TIME_H -#include <compat.h> +#include <compat/compat.h> #include <chrono> #include <cstdint> @@ -24,6 +24,7 @@ struct NodeClock : public std::chrono::system_clock { }; using NodeSeconds = std::chrono::time_point<NodeClock, std::chrono::seconds>; +using SteadyClock = std::chrono::steady_clock; using SteadySeconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::seconds>; using SteadyMilliseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::milliseconds>; using SteadyMicroseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>; @@ -54,6 +55,7 @@ constexpr int64_t count_seconds(std::chrono::seconds t) { return t.count(); } constexpr int64_t count_milliseconds(std::chrono::milliseconds t) { return t.count(); } constexpr int64_t count_microseconds(std::chrono::microseconds t) { return t.count(); } +using HoursDouble = std::chrono::duration<double, std::chrono::hours::period>; using SecondsDouble = std::chrono::duration<double, std::chrono::seconds::period>; /** diff --git a/src/validation.cpp b/src/validation.cpp index 4c694a2c21..17211956f5 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,6 +5,9 @@ #include <validation.h> +#include <kernel/coinstats.h> +#include <kernel/mempool_persist.h> + #include <arith_uint256.h> #include <chain.h> #include <chainparams.h> @@ -17,8 +20,8 @@ #include <consensus/validation.h> #include <cuckoocache.h> #include <flatfile.h> +#include <fs.h> #include <hash.h> -#include <kernel/coinstats.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -47,12 +50,15 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <warnings.h> #include <algorithm> +#include <cassert> +#include <chrono> #include <deque> #include <numeric> #include <optional> @@ -61,8 +67,9 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; +using kernel::LoadMempool; -using node::BLOCKFILE_CHUNK_SIZE; +using fsbridge::FopenFn; using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; @@ -70,11 +77,8 @@ using node::CBlockIndexWorkComparator; using node::fImporting; using node::fPruneMode; using node::fReindex; -using node::nPruneTarget; -using node::OpenBlockFile; using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UNDOFILE_CHUNK_SIZE; using node::UndoReadFromDisk; using node::UnlinkPrunedFiles; @@ -2263,7 +2267,6 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, m_blockman.m_dirty_blockindex.insert(pindex); } - assert(pindex->phashBlock); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3861,13 +3864,11 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void CChainState::LoadMempool(const ArgsManager& args) +void CChainState::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) { if (!m_mempool) return; - if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(*m_mempool, *this); - } - m_mempool->SetIsLoaded(!ShutdownRequested()); + ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); + m_mempool->SetLoadTried(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4256,11 +4257,16 @@ bool CChainState::LoadGenesisBlock() return true; } -void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) +void CChainState::LoadExternalBlockFile( + FILE* fileIn, + FlatFilePos* dbp, + std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent) { AssertLockNotHeld(m_chainstate_mutex); - // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; + + // Either both should be specified (-reindex), or neither (-loadblock). + assert(!dbp == !blocks_with_unknown_parent); + int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4310,8 +4316,9 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); - if (dbp) - mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); + if (dbp && blocks_with_unknown_parent) { + blocks_with_unknown_parent->emplace(block.hashPrevBlock, *dbp); + } continue; } @@ -4340,13 +4347,15 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) NotifyHeaderTip(*this); + if (!blocks_with_unknown_parent) continue; + // Recursively process earlier encountered successors of this block std::deque<uint256> queue; queue.push_back(hash); while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + auto range = blocks_with_unknown_parent->equal_range(head); while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); @@ -4361,7 +4370,7 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } } range.first++; - mapBlocksUnknownParent.erase(it); + blocks_with_unknown_parent->erase(it); NotifyHeaderTip(*this); } } @@ -4638,153 +4647,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -static const uint64_t MEMPOOL_DUMP_VERSION = 1; - -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) -{ - int64_t nExpiryTimeout = std::chrono::seconds{pool.m_expiry}.count(); - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); - return false; - } - - int64_t count = 0; - int64_t expired = 0; - int64_t failed = 0; - int64_t already_there = 0; - int64_t unbroadcast = 0; - int64_t nNow = GetTime(); - - try { - uint64_t version; - file >> version; - if (version != MEMPOOL_DUMP_VERSION) { - return false; - } - uint64_t num; - file >> num; - while (num) { - --num; - CTransactionRef tx; - int64_t nTime; - int64_t nFeeDelta; - file >> tx; - file >> nTime; - file >> nFeeDelta; - - CAmount amountdelta = nFeeDelta; - if (amountdelta) { - pool.PrioritiseTransaction(tx->GetHash(), amountdelta); - } - if (nTime > nNow - nExpiryTimeout) { - LOCK(cs_main); - const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); - if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ++count; - } else { - // mempool may contain the transaction already, e.g. from - // wallet(s) having loaded it while we were processing - // mempool transactions; consider these as valid, instead of - // failed, but mark them as 'already there' - if (pool.exists(GenTxid::Txid(tx->GetHash()))) { - ++already_there; - } else { - ++failed; - } - } - } else { - ++expired; - } - if (ShutdownRequested()) - return false; - } - std::map<uint256, CAmount> mapDeltas; - file >> mapDeltas; - - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); - } - - std::set<uint256> unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); - } - } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); - return false; - } - - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); - return true; -} - -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function, bool skip_file_commit) -{ - int64_t start = GetTimeMicros(); - - std::map<uint256, CAmount> mapDeltas; - std::vector<TxMempoolInfo> vinfo; - std::set<uint256> unbroadcast_txids; - - static Mutex dump_mutex; - LOCK(dump_mutex); - - { - LOCK(pool.cs); - for (const auto &i : pool.mapDeltas) { - mapDeltas[i.first] = i.second; - } - vinfo = pool.infoAll(); - unbroadcast_txids = pool.GetUnbroadcastTxs(); - } - - int64_t mid = GetTimeMicros(); - - try { - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat.new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - - uint64_t version = MEMPOOL_DUMP_VERSION; - file << version; - - file << (uint64_t)vinfo.size(); - for (const auto& i : vinfo) { - file << *(i.tx); - file << int64_t{count_seconds(i.m_time)}; - file << int64_t{i.nFeeDelta}; - mapDeltas.erase(i.tx->GetHash()); - } - - file << mapDeltas; - - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); - file << unbroadcast_txids; - - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); - file.fclose(); - if (!RenameOver(gArgs.GetDataDirNet() / "mempool.dat.new", gArgs.GetDataDirNet() / "mempool.dat")) { - throw std::runtime_error("Rename failed"); - } - int64_t last = GetTimeMicros(); - LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); - } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); - return false; - } - return true; -} - //! Guess how far we are in the verification process at the given block index //! require cs_main if pindex has not been validated yet (because nChainTx might be unset) double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { @@ -4867,7 +4729,7 @@ const AssumeutxoData* ExpectedAssumeutxo( } bool ChainstateManager::ActivateSnapshot( - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory) { @@ -4962,7 +4824,7 @@ static void FlushSnapshotToDisk(CCoinsViewCache& coins_cache, bool snapshot_load bool ChainstateManager::PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const SnapshotMetadata& metadata) { // It's okay to release cs_main before we're done using `coins_cache` because we know diff --git a/src/validation.h b/src/validation.h index 0e27e117fa..9fef69799b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -68,8 +68,6 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -/** Default for -persistmempool */ -static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -577,8 +575,36 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** Import blocks from an external file */ - void LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp = nullptr) + /** + * Import blocks from an external file + * + * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). + * It reads all blocks contained in the given file and attempts to process them (add them to the + * block index). The blocks may be out of order within each file and across files. Often this + * function reads a block but finds that its parent hasn't been read yet, so the block can't be + * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is + * passed as an argument), so that when the block's parent is later read and processed, this + * function can re-read the child block from disk and process it. + * + * Because a block's parent may be in a later file, not just later in the same file, the + * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, + * rather than just a map, because multiple blocks may have the same parent (when chain splits + * or stale blocks exist). It maps from parent-hash to child-disk-position. + * + * This function can also be used to read blocks from user-specified block files using the + * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. + * + * + * @param[in] fileIn FILE handle to file containing blocks to read + * @param[in] dbp (optional) Disk block position (only for reindex) + * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with + * unknown parent, key is parent block hash + * (only used for reindex) + * */ + void LoadExternalBlockFile( + FILE* fileIn, + FlatFilePos* dbp = nullptr, + std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr) EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex); /** @@ -679,7 +705,7 @@ public: void CheckBlockIndex(); /** Load the persisted mempool from disk */ - void LoadMempool(const ArgsManager& args); + void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -820,7 +846,7 @@ private: //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( CChainState& snapshot_chainstate, - CAutoFile& coins_file, + AutoFile& coins_file, const node::SnapshotMetadata& metadata); /** @@ -834,7 +860,7 @@ private: friend CChainState; public: - using Options = ChainstateManagerOpts; + using Options = kernel::ChainstateManagerOpts; explicit ChainstateManager(const Options& opts) : m_chainparams{opts.chainparams}, @@ -909,7 +935,7 @@ public: //! - Move the new chainstate to `m_snapshot_chainstate` and make it our //! ChainstateActive(). [[nodiscard]] bool ActivateSnapshot( - CAutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); + AutoFile& coins_file, const node::SnapshotMetadata& metadata, bool in_memory); //! The most-work chain. CChainState& ActiveChainstate() const; @@ -1014,14 +1040,6 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) return DeploymentEnabled(chainman.GetConsensus(), dep); } -using FopenFn = std::function<FILE*(const fs::path&, const char*)>; - -/** Dump the mempool to disk. */ -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); - -/** Load the mempool from disk. */ -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index dbd768a758..60715ff3c8 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <compat/compat.h> #include <fs.h> #include <wallet/bdb.h> #include <wallet/db.h> diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index d80c3e25b0..f7fee443d0 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -41,7 +41,7 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) return false; } - CHashWriter hasher(0, 0); + HashWriter hasher{}; WalletDatabase& db = wallet.GetDatabase(); std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(); @@ -132,7 +132,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: std::ifstream dump_file{dump_path}; // Compute the checksum - CHashWriter hasher(0, 0); + HashWriter hasher{}; uint256 checksum; // Check the magic and version diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 547d972c8d..23f91d9b3a 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -569,11 +569,13 @@ public: options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } - std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) override + BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) override { DatabaseStatus status; - - return MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings)); + bilingual_str error; + BResult<std::unique_ptr<Wallet>> wallet{MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings))}; + if (!wallet) return error; + return wallet; } std::string getWalletDir() override { diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371..944925d600 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -9,15 +9,12 @@ #include <wallet/wallet.h> namespace wallet { -isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) +isminetype InputIsMine(const CWallet& wallet, const CTxIn& txin) { AssertLockHeld(wallet.cs_wallet); - std::map<uint256, CWalletTx>::const_iterator mi = wallet.mapWallet.find(txin.prevout.hash); - if (mi != wallet.mapWallet.end()) - { - const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.tx->vout.size()) - return wallet.IsMine(prev.tx->vout[txin.prevout.n]); + const CWalletTx* prev = wallet.GetWalletTx(txin.prevout.hash); + if (prev && txin.prevout.n < prev->tx->vout.size()) { + return wallet.IsMine(prev->tx->vout[txin.prevout.n]); } return ISMINE_NO; } @@ -25,20 +22,8 @@ isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) bool AllInputsMine(const CWallet& wallet, const CTransaction& tx, const isminefilter& filter) { LOCK(wallet.cs_wallet); - - for (const CTxIn& txin : tx.vin) - { - auto mi = wallet.mapWallet.find(txin.prevout.hash); - if (mi == wallet.mapWallet.end()) - return false; // any unknown inputs can't be from us - - const CWalletTx& prev = (*mi).second; - - if (txin.prevout.n >= prev.tx->vout.size()) - return false; // invalid input! - - if (!(wallet.IsMine(prev.tx->vout[txin.prevout.n]) & filter)) - return false; + for (const CTxIn& txin : tx.vin) { + if (!(InputIsMine(wallet, txin) & filter)) return false; } return true; } @@ -111,10 +96,10 @@ CAmount TxGetChange(const CWallet& wallet, const CTransaction& tx) return nChange; } -static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter, bool recalculate = false) +static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter) { auto& amount = wtx.m_amounts[type]; - if (recalculate || !amount.m_cached[filter]) { + if (!amount.m_cached[filter]) { amount.Set(filter, type == CWalletTx::DEBIT ? wallet.GetDebit(*wtx.tx, filter) : TxGetCredit(wallet, *wtx.tx, filter)); wtx.m_is_cache_empty = false; } @@ -130,12 +115,10 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) { + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, get_amount_filter); } return credit; } @@ -146,11 +129,9 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi return 0; CAmount debit = 0; - if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, get_amount_filter); } return debit; } @@ -164,29 +145,18 @@ CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) return wtx.nChangeCached; } -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache) -{ - AssertLockHeld(wallet.cs_wallet); - - if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); - } - - return 0; -} - -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { - return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, filter); } return 0; } -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache, const isminefilter& filter) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { AssertLockHeld(wallet.cs_wallet); @@ -197,7 +167,7 @@ CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, if (wallet.IsTxImmatureCoinBase(wtx)) return 0; - if (fUseCache && allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { + if (allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { return wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_value[filter]; } @@ -332,8 +302,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) const CWalletTx& wtx = entry.second; const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; - const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /*fUseCache=*/true, ISMINE_WATCH_ONLY | reuse_filter)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; @@ -342,8 +312,8 @@ Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx); - ret.m_watchonly_immature += CachedTxGetImmatureWatchOnlyCredit(wallet, wtx); + ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE); + ret.m_watchonly_immature += CachedTxGetImmatureCredit(wallet, wtx, ISMINE_WATCH_ONLY); } } return ret; diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 1caef293f2..41a70b089a 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -29,11 +29,9 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism //! filter decides which addresses will count towards the debit CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); -CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true) +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true) - EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); -CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter = ISMINE_SPENDABLE) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); struct COutputEntry { diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 9428d049de..2a6eb96871 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -34,7 +34,7 @@ RPCHelpMan getnewaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -86,7 +86,7 @@ RPCHelpMan getrawchangeaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -131,7 +131,7 @@ RPCHelpMan setlabel() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -148,7 +148,7 @@ RPCHelpMan setlabel() pwallet->SetAddressBook(dest, label, "send"); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -181,7 +181,7 @@ RPCHelpMan listaddressgroupings() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -252,7 +252,7 @@ RPCHelpMan addmultisigaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); @@ -327,7 +327,7 @@ RPCHelpMan keypoolrefill() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; if (pwallet->IsLegacy() && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); @@ -350,7 +350,7 @@ RPCHelpMan keypoolrefill() throw JSONRPCError(RPC_WALLET_ERROR, "Error refreshing keypool."); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -374,14 +374,14 @@ RPCHelpMan newkeypool() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); spk_man.NewKeyPool(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -548,7 +548,7 @@ RPCHelpMan getaddressinfo() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -658,7 +658,7 @@ RPCHelpMan getaddressesbylabel() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -721,7 +721,7 @@ RPCHelpMan listlabels() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -763,7 +763,7 @@ RPCHelpMan walletdisplayaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - if (!wallet) return NullUniValue; + if (!wallet) return UniValue::VNULL; CWallet* const pwallet = wallet.get(); LOCK(pwallet->cs_wallet); diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 5f1673fb12..306053fd0c 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -100,11 +100,13 @@ RPCHelpMan importprivkey() "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, RPCArg::DefaultHint{"current label if address exists, otherwise \"\""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -122,7 +124,7 @@ RPCHelpMan importprivkey() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); @@ -138,7 +140,7 @@ RPCHelpMan importprivkey() EnsureWalletIsUnlocked(*pwallet); std::string strSecret = request.params[0].get_str(); - std::string strLabel = ""; + std::string strLabel; if (!request.params[1].isNull()) strLabel = request.params[1].get_str(); @@ -190,7 +192,7 @@ RPCHelpMan importprivkey() RescanWallet(*pwallet, reserver); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -201,6 +203,8 @@ RPCHelpMan importaddress() "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" @@ -210,7 +214,7 @@ RPCHelpMan importaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, {"p2sh", RPCArg::Type::BOOL, RPCArg::Default{false}, "Add the P2SH version of the script as well"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, @@ -225,7 +229,7 @@ RPCHelpMan importaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; EnsureLegacyScriptPubKeyMan(*pwallet, true); @@ -295,7 +299,7 @@ RPCHelpMan importaddress() } } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -313,7 +317,7 @@ RPCHelpMan importprunedfunds() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; CMutableTransaction tx; if (!DecodeHexTx(tx, request.params[0].get_str())) { @@ -348,7 +352,7 @@ RPCHelpMan importprunedfunds() CTransactionRef tx_ref = MakeTransactionRef(tx); if (pwallet->IsMine(*tx_ref)) { pwallet->AddToWallet(std::move(tx_ref), TxStateConfirmed{merkleBlock.header.GetHash(), height, static_cast<int>(txnIndex)}); - return NullUniValue; + return UniValue::VNULL; } throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No addresses in wallet correspond to included transaction"); @@ -372,7 +376,7 @@ RPCHelpMan removeprunedfunds() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -389,7 +393,7 @@ RPCHelpMan removeprunedfunds() throw JSONRPCError(RPC_INVALID_PARAMETER, "Transaction does not exist in wallet."); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -401,11 +405,13 @@ RPCHelpMan importpubkey() "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -419,7 +425,7 @@ RPCHelpMan importpubkey() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; EnsureLegacyScriptPubKeyMan(*pwallet, true); @@ -474,7 +480,7 @@ RPCHelpMan importpubkey() } } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -484,7 +490,7 @@ RPCHelpMan importwallet() { return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Blockchain and Mempool will be rescanned after a successful import. Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -500,7 +506,7 @@ RPCHelpMan importwallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; EnsureLegacyScriptPubKeyMan(*pwallet, true); @@ -631,7 +637,7 @@ RPCHelpMan importwallet() if (!fGood) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys/scripts to wallet"); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -655,7 +661,7 @@ RPCHelpMan dumpprivkey() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(*pwallet); @@ -705,7 +711,7 @@ RPCHelpMan dumpwallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; const CWallet& wallet = *pwallet; const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(wallet); @@ -1250,6 +1256,8 @@ RPCHelpMan importmulti() "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", @@ -1291,7 +1299,7 @@ RPCHelpMan importmulti() "\"requests\""}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, "\"options\""}, }, @@ -1320,7 +1328,7 @@ RPCHelpMan importmulti() [&](const RPCHelpMan& self, const JSONRPCRequest& mainRequest) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(mainRequest); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; CWallet& wallet{*pwallet}; // Make sure the results are valid at least up to the most recent block @@ -1593,7 +1601,7 @@ RPCHelpMan importdescriptors() " Use the string \"now\" to substitute the current synced blockchain time.\n" " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" - " of all descriptors being imported will be scanned.", + "of all descriptors being imported will be scanned as well as the mempool.", /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, @@ -1628,7 +1636,7 @@ RPCHelpMan importdescriptors() [&](const RPCHelpMan& self, const JSONRPCRequest& main_request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(main_request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; CWallet& wallet{*pwallet}; // Make sure the results are valid at least up to the most recent block @@ -1763,7 +1771,7 @@ RPCHelpMan listdescriptors() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> wallet = GetWalletForJSONRPCRequest(request); - if (!wallet) return NullUniValue; + if (!wallet) return UniValue::VNULL; if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { throw JSONRPCError(RPC_WALLET_ERROR, "listdescriptors is not available for non-descriptor wallets"); @@ -1832,7 +1840,7 @@ RPCHelpMan backupwallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -1845,7 +1853,7 @@ RPCHelpMan backupwallet() throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); } - return NullUniValue; + return UniValue::VNULL; }, }; } diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index a9fff95882..d40d800f28 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -103,7 +103,7 @@ RPCHelpMan getreceivedbyaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -144,7 +144,7 @@ RPCHelpMan getreceivedbylabel() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -184,7 +184,7 @@ RPCHelpMan getbalance() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -223,7 +223,7 @@ RPCHelpMan getunconfirmedbalance() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -282,7 +282,7 @@ RPCHelpMan lockunspent() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -407,7 +407,7 @@ RPCHelpMan listlockunspent() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -459,7 +459,7 @@ RPCHelpMan getbalances() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> rpc_wallet = GetWalletForJSONRPCRequest(request); - if (!rpc_wallet) return NullUniValue; + if (!rpc_wallet) return UniValue::VNULL; const CWallet& wallet = *rpc_wallet; // Make sure the results are valid at least up to the most recent block @@ -559,7 +559,7 @@ RPCHelpMan listunspent() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; int nMinDepth = 1; if (!request.params[0].isNull()) { @@ -638,7 +638,7 @@ RPCHelpMan listunspent() cctl.m_max_depth = nMaxDepth; cctl.m_include_unsafe_inputs = include_unsafe; LOCK(pwallet->cs_wallet); - vecOutputs = AvailableCoinsListUnspent(*pwallet, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount).coins; + vecOutputs = AvailableCoinsListUnspent(*pwallet, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount).all(); } LOCK(pwallet->cs_wallet); diff --git a/src/wallet/rpc/encrypt.cpp b/src/wallet/rpc/encrypt.cpp index 931c64ca06..a68f52a718 100644 --- a/src/wallet/rpc/encrypt.cpp +++ b/src/wallet/rpc/encrypt.cpp @@ -32,7 +32,7 @@ RPCHelpMan walletpassphrase() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - if (!wallet) return NullUniValue; + if (!wallet) return UniValue::VNULL; CWallet* const pwallet = wallet.get(); int64_t nSleepTime; @@ -98,7 +98,7 @@ RPCHelpMan walletpassphrase() } }, nSleepTime); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -120,7 +120,7 @@ RPCHelpMan walletpassphrasechange() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -146,7 +146,7 @@ RPCHelpMan walletpassphrasechange() throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -173,7 +173,7 @@ RPCHelpMan walletlock() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -184,7 +184,7 @@ RPCHelpMan walletlock() pwallet->Lock(); pwallet->nRelockTime = 0; - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -217,7 +217,7 @@ RPCHelpMan encryptwallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); diff --git a/src/wallet/rpc/signmessage.cpp b/src/wallet/rpc/signmessage.cpp index 438d290030..ae4bd4fbc5 100644 --- a/src/wallet/rpc/signmessage.cpp +++ b/src/wallet/rpc/signmessage.cpp @@ -36,7 +36,7 @@ RPCHelpMan signmessage() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 22c1f2bdc2..e27cb1139e 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -261,7 +261,7 @@ RPCHelpMan sendtoaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -367,7 +367,7 @@ RPCHelpMan sendmany() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -422,7 +422,7 @@ RPCHelpMan settxfee() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LOCK(pwallet->cs_wallet); @@ -805,7 +805,7 @@ RPCHelpMan fundrawtransaction() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType(), UniValue::VBOOL}); @@ -895,7 +895,7 @@ RPCHelpMan signrawtransactionwithwallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR, UniValue::VSTR}, true); @@ -992,7 +992,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) [want_psbt](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); @@ -1209,7 +1209,7 @@ RPCHelpMan send() ); std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; UniValue options{request.params[4].isNull() ? UniValue::VOBJ : request.params[4]}; InterpretFeeEstimationInstructions(/*conf_target=*/request.params[1], /*estimate_mode=*/request.params[2], /*fee_rate=*/request.params[3], options); @@ -1314,7 +1314,7 @@ RPCHelpMan sendall() ); std::shared_ptr<CWallet> const pwallet{GetWalletForJSONRPCRequest(request)}; - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); @@ -1382,7 +1382,7 @@ RPCHelpMan sendall() total_input_value += tx->tx->vout[input.prevout.n].nValue; } } else { - for (const COutput& output : AvailableCoins(*pwallet, &coin_control, fee_rate, /*nMinimumAmount=*/0).coins) { + for (const COutput& output : AvailableCoins(*pwallet, &coin_control, fee_rate, /*nMinimumAmount=*/0).all()) { CHECK_NONFATAL(output.input_bytes > 0); if (send_max && fee_rate.GetFee(output.input_bytes) > output.txout.nValue) { continue; @@ -1490,7 +1490,7 @@ RPCHelpMan walletprocesspsbt() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; const CWallet& wallet{*pwallet}; // Make sure the results are valid at least up to the most recent block @@ -1617,7 +1617,7 @@ RPCHelpMan walletcreatefundedpsbt() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; CWallet& wallet{*pwallet}; // Make sure the results are valid at least up to the most recent block @@ -1633,7 +1633,7 @@ RPCHelpMan walletcreatefundedpsbt() }, true ); - UniValue options = request.params[3]; + UniValue options{request.params[3].isNull() ? UniValue::VOBJ : request.params[3]}; CAmount fee; int change_position; diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 0b52dcc001..9c8003d6d7 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -236,7 +236,7 @@ RPCHelpMan listreceivedbyaddress() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -281,7 +281,7 @@ RPCHelpMan listreceivedbylabel() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -471,7 +471,7 @@ RPCHelpMan listtransactions() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -586,7 +586,7 @@ RPCHelpMan listsinceblock() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; const CWallet& wallet = *pwallet; // Make sure the results are valid at least up to the most recent block @@ -727,7 +727,7 @@ RPCHelpMan gettransaction() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -800,7 +800,7 @@ RPCHelpMan abandontransaction() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -817,7 +817,7 @@ RPCHelpMan abandontransaction() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not eligible for abandonment"); } - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -845,7 +845,7 @@ RPCHelpMan rescanblockchain() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; CWallet& wallet{*pwallet}; // Make sure the results are valid at least up to the most recent block @@ -925,7 +925,7 @@ RPCHelpMan abortrescan() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; pwallet->AbortRescan(); diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 4c44c333a6..0fe8871152 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -68,7 +68,7 @@ static RPCHelpMan getwalletinfo() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -241,7 +241,7 @@ static RPCHelpMan loadwallet() static RPCHelpMan setwalletflag() { - std::string flags = ""; + std::string flags; for (auto& it : WALLET_FLAG_MAP) if (it.second & MUTABLE_WALLET_FLAGS) flags += (flags == "" ? "" : ", ") + it.first; @@ -267,7 +267,7 @@ static RPCHelpMan setwalletflag() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; std::string flag_str = request.params[0].get_str(); bool value = request.params[1].isNull() || request.params[1].get_bool(); @@ -480,7 +480,7 @@ static RPCHelpMan sethdseed() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); @@ -521,7 +521,7 @@ static RPCHelpMan sethdseed() spk_man.SetHDSeed(master_pub_key); if (flush_key_pool) spk_man.NewKeyPool(); - return NullUniValue; + return UniValue::VNULL; }, }; } @@ -532,7 +532,7 @@ static RPCHelpMan upgradewallet() "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified.\n" "New keys may be generated and a new wallet backup will need to be made.", { - {"version", RPCArg::Type::NUM, RPCArg::Default{FEATURE_LATEST}, "The version number to upgrade to. Default is the latest wallet version."} + {"version", RPCArg::Type::NUM, RPCArg::Default{int{FEATURE_LATEST}}, "The version number to upgrade to. Default is the latest wallet version."} }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -551,7 +551,7 @@ static RPCHelpMan upgradewallet() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; + if (!pwallet) return UniValue::VNULL; RPCTypeCheck(request.params, {UniValue::VNUM}, true); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 9be29c4709..c00a2cd023 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -79,6 +79,32 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle return CalculateMaximumSignedTxSize(tx, wallet, txouts, coin_control); } +uint64_t CoinsResult::size() const +{ + return bech32m.size() + bech32.size() + P2SH_segwit.size() + legacy.size() + other.size(); +} + +std::vector<COutput> CoinsResult::all() const +{ + std::vector<COutput> all; + all.reserve(this->size()); + all.insert(all.end(), bech32m.begin(), bech32m.end()); + all.insert(all.end(), bech32.begin(), bech32.end()); + all.insert(all.end(), P2SH_segwit.begin(), P2SH_segwit.end()); + all.insert(all.end(), legacy.begin(), legacy.end()); + all.insert(all.end(), other.begin(), other.end()); + return all; +} + +void CoinsResult::clear() +{ + bech32m.clear(); + bech32.clear(); + P2SH_segwit.clear(); + legacy.clear(); + other.clear(); +} + CoinsResult AvailableCoins(const CWallet& wallet, const CCoinControl* coinControl, std::optional<CFeeRate> feerate, @@ -193,10 +219,55 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; + // When parsing a scriptPubKey, Solver returns the parsed pubkeys or hashes (depending on the script) + // We don't need those here, so we are leaving them in return_values_unused + std::vector<std::vector<uint8_t>> return_values_unused; + TxoutType type; + bool is_from_p2sh{false}; + + // If the Output is P2SH and spendable, we want to know if it is + // a P2SH (legacy) or one of P2SH-P2WPKH, P2SH-P2WSH (P2SH-Segwit). We can determine + // this from the redeemScript. If the Output is not spendable, it will be classified + // as a P2SH (legacy), since we have no way of knowing otherwise without the redeemScript + if (output.scriptPubKey.IsPayToScriptHash() && solvable) { + CScript redeemScript; + CTxDestination destination; + if (!ExtractDestination(output.scriptPubKey, destination)) + continue; + const CScriptID& hash = CScriptID(std::get<ScriptHash>(destination)); + if (!provider->GetCScript(hash, redeemScript)) + continue; + type = Solver(redeemScript, return_values_unused); + is_from_p2sh = true; + } else { + type = Solver(output.scriptPubKey, return_values_unused); + } + int input_bytes = CalculateMaximumSignedInputSize(output, COutPoint(), provider.get(), coinControl); - result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); + COutput coin(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); + switch (type) { + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V1_TAPROOT: + result.bech32m.push_back(coin); + break; + case TxoutType::WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: + if (is_from_p2sh) { + result.P2SH_segwit.push_back(coin); + break; + } + result.bech32.push_back(coin); + break; + case TxoutType::SCRIPTHASH: + case TxoutType::PUBKEYHASH: + result.legacy.push_back(coin); + break; + default: + result.other.push_back(coin); + }; + + // Cache total amount as we go result.total_amount += output.nValue; - // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { if (result.total_amount >= nMinimumSumAmount) { @@ -205,7 +276,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, } // Checks the maximum number of UTXO's. - if (nMaximumCount > 0 && result.coins.size() >= nMaximumCount) { + if (nMaximumCount > 0 && result.size() >= nMaximumCount) { return result; } } @@ -261,7 +332,7 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) std::map<CTxDestination, std::vector<COutput>> result; - for (const COutput& coin : AvailableCoinsListUnspent(wallet).coins) { + for (const COutput& coin : AvailableCoinsListUnspent(wallet).all()) { CTxDestination address; if ((coin.spendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.solvable)) && ExtractDestination(FindNonChangeParentOutput(wallet, coin.outpoint).scriptPubKey, address)) { @@ -379,20 +450,51 @@ std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<C return groups_out; } -std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, - const CoinSelectionParams& coin_selection_params) +std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, const CoinsResult& available_coins, + const CoinSelectionParams& coin_selection_params, bool allow_mixed_output_types) +{ + // Run coin selection on each OutputType and compute the Waste Metric + std::vector<SelectionResult> results; + if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.legacy, coin_selection_params)}) { + results.push_back(*result); + } + if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.P2SH_segwit, coin_selection_params)}) { + results.push_back(*result); + } + if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.bech32, coin_selection_params)}) { + results.push_back(*result); + } + if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.bech32m, coin_selection_params)}) { + results.push_back(*result); + } + + // If we can't fund the transaction from any individual OutputType, run coin selection + // over all available coins, else pick the best solution from the results + if (results.size() == 0) { + if (allow_mixed_output_types) { + if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.all(), coin_selection_params)}) { + return result; + } + } + return std::optional<SelectionResult>(); + }; + std::optional<SelectionResult> result{*std::min_element(results.begin(), results.end())}; + return result; +}; + +std::optional<SelectionResult> ChooseSelectionResult(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, const std::vector<COutput>& available_coins, const CoinSelectionParams& coin_selection_params) { // Vector of results. We will choose the best one based on waste. std::vector<SelectionResult> results; // Note that unlike KnapsackSolver, we do not include the fee for creating a change output as BnB will not create a change output. - std::vector<OutputGroup> positive_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, true /* positive_only */); + std::vector<OutputGroup> positive_groups = GroupOutputs(wallet, available_coins, coin_selection_params, eligibility_filter, true /* positive_only */); if (auto bnb_result{SelectCoinsBnB(positive_groups, nTargetValue, coin_selection_params.m_cost_of_change)}) { results.push_back(*bnb_result); } // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. - std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); + std::vector<OutputGroup> all_groups = GroupOutputs(wallet, available_coins, coin_selection_params, eligibility_filter, false /* positive_only */); CAmount target_with_change = nTargetValue; // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. // So we need to include that for KnapsackSolver and SRD as well, as we are expecting to create a change output. @@ -425,9 +527,8 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm return best_result; } -std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) +std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const CAmount& nTargetValue, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) { - std::vector<COutput> vCoins(vAvailableCoins); CAmount value_to_select = nTargetValue; OutputGroup preset_inputs(coin_selection_params); @@ -487,13 +588,13 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec return result; } - // remove preset inputs from vCoins so that Coin Selection doesn't pick them. - for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coin_control.HasSelected();) - { - if (preset_coins.count(it->outpoint)) - it = vCoins.erase(it); - else - ++it; + // remove preset inputs from coins so that Coin Selection doesn't pick them. + if (coin_control.HasSelected()) { + available_coins.legacy.erase(remove_if(available_coins.legacy.begin(), available_coins.legacy.end(), [&](const COutput& c) { return preset_coins.count(c.outpoint); }), available_coins.legacy.end()); + available_coins.P2SH_segwit.erase(remove_if(available_coins.P2SH_segwit.begin(), available_coins.P2SH_segwit.end(), [&](const COutput& c) { return preset_coins.count(c.outpoint); }), available_coins.P2SH_segwit.end()); + available_coins.bech32.erase(remove_if(available_coins.bech32.begin(), available_coins.bech32.end(), [&](const COutput& c) { return preset_coins.count(c.outpoint); }), available_coins.bech32.end()); + available_coins.bech32m.erase(remove_if(available_coins.bech32m.begin(), available_coins.bech32m.end(), [&](const COutput& c) { return preset_coins.count(c.outpoint); }), available_coins.bech32m.end()); + available_coins.other.erase(remove_if(available_coins.other.begin(), available_coins.other.end(), [&](const COutput& c) { return preset_coins.count(c.outpoint); }), available_coins.other.end()); } unsigned int limit_ancestor_count = 0; @@ -505,11 +606,15 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included - if (coin_control.m_avoid_partial_spends && vCoins.size() > OUTPUT_GROUP_MAX_ENTRIES) { + if (coin_control.m_avoid_partial_spends && available_coins.size() > OUTPUT_GROUP_MAX_ENTRIES) { // Cases where we have 101+ outputs all pointing to the same destination may result in // privacy leaks as they will potentially be deterministically sorted. We solve that by // explicitly shuffling the outputs before processing - Shuffle(vCoins.begin(), vCoins.end(), coin_selection_params.rng_fast); + Shuffle(available_coins.legacy.begin(), available_coins.legacy.end(), coin_selection_params.rng_fast); + Shuffle(available_coins.P2SH_segwit.begin(), available_coins.P2SH_segwit.end(), coin_selection_params.rng_fast); + Shuffle(available_coins.bech32.begin(), available_coins.bech32.end(), coin_selection_params.rng_fast); + Shuffle(available_coins.bech32m.begin(), available_coins.bech32m.end(), coin_selection_params.rng_fast); + Shuffle(available_coins.other.begin(), available_coins.other.end(), coin_selection_params.rng_fast); } // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the @@ -521,26 +626,27 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six // confirmations on outputs received from other wallets and only spend confirmed change. - if (auto r1{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, coin_selection_params)}) return r1; - if (auto r2{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, coin_selection_params)}) return r2; + if (auto r1{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 6, 0), available_coins, coin_selection_params, /*allow_mixed_output_types=*/false)}) return r1; + // Allow mixing only if no solution from any single output type can be found + if (auto r2{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 1, 0), available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) return r2; // Fall back to using zero confirmation change (but with as few ancestors in the mempool as // possible) if we cannot fund the transaction otherwise. if (wallet.m_spend_zero_conf_change) { - if (auto r3{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, coin_selection_params)}) return r3; + if (auto r3{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, 2), available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) return r3; if (auto r4{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), - vCoins, coin_selection_params)}) { + available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r4; } if (auto r5{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), - vCoins, coin_selection_params)}) { + available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r5; } // If partial groups are allowed, relax the requirement of spending OutputGroups (groups // of UTXOs sent to the same address, which are obviously controlled by a single wallet) // in their entirety. if (auto r6{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), - vCoins, coin_selection_params)}) { + available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r6; } // Try with unsafe inputs if they are allowed. This may spend unconfirmed outputs @@ -548,7 +654,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (coin_control.m_include_unsafe_inputs) { if (auto r7{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0 /* conf_mine */, 0 /* conf_theirs */, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), - vCoins, coin_selection_params)}) { + available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r7; } } @@ -558,7 +664,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (!fRejectLongChains) { if (auto r8{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), - vCoins, coin_selection_params)}) { + available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r8; } } @@ -778,7 +884,7 @@ static BResult<CreatedTransactionResult> CreateTransactionInternal( CAmount selection_target = recipients_sum + not_input_fees; // Get available coins - auto res_available_coins = AvailableCoins(wallet, + auto available_coins = AvailableCoins(wallet, &coin_control, coin_selection_params.m_effective_feerate, 1, /*nMinimumAmount*/ @@ -787,7 +893,7 @@ static BResult<CreatedTransactionResult> CreateTransactionInternal( 0); /*nMaximumCount*/ // Choose coins to use - std::optional<SelectionResult> result = SelectCoins(wallet, res_available_coins.coins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); + std::optional<SelectionResult> result = SelectCoins(wallet, available_coins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); if (!result) { return _("Insufficient funds"); } diff --git a/src/wallet/spend.h b/src/wallet/spend.h index ab0ff1ee58..96ecd690be 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -29,14 +29,38 @@ struct TxSize { TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control = nullptr); TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); +/** + * COutputs available for spending, stored by OutputType. + * This struct is really just a wrapper around OutputType vectors with a convenient + * method for concatenating and returning all COutputs as one vector. + * + * clear(), size() methods are implemented so that one can interact with + * the CoinsResult struct as if it was a vector + */ struct CoinsResult { - std::vector<COutput> coins; - // Sum of all the coins amounts + /** Vectors for each OutputType */ + std::vector<COutput> legacy; + std::vector<COutput> P2SH_segwit; + std::vector<COutput> bech32; + std::vector<COutput> bech32m; + + /** Other is a catch-all for anything that doesn't match the known OutputTypes */ + std::vector<COutput> other; + + /** Concatenate and return all COutputs as one vector */ + std::vector<COutput> all() const; + + /** The following methods are provided so that CoinsResult can mimic a vector, + * i.e., methods can work with individual OutputType vectors or on the entire object */ + uint64_t size() const; + void clear(); + + /** Sum of all available coins */ CAmount total_amount{0}; }; + /** - * Return vector of available COutputs. - * By default, returns only the spendable coins. + * Populate the CoinsResult struct with vectors of available COutputs, organized by OutputType. */ CoinsResult AvailableCoins(const CWallet& wallet, const CCoinControl* coinControl = nullptr, @@ -67,35 +91,52 @@ const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const COutPoint& std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<COutput>& outputs, const CoinSelectionParams& coin_sel_params, const CoinEligibilityFilter& filter, bool positive_only); +/** + * Attempt to find a valid input set that preserves privacy by not mixing OutputTypes. + * `ChooseSelectionResult()` will be called on each OutputType individually and the best + * the solution (according to the waste metric) will be chosen. If a valid input cannot be found from any + * single OutputType, fallback to running `ChooseSelectionResult()` over all available coins. + * + * param@[in] wallet The wallet which provides solving data for the coins + * param@[in] nTargetValue The target value + * param@[in] eligilibity_filter A filter containing rules for which coins are allowed to be included in this selection + * param@[in] available_coins The struct of coins, organized by OutputType, available for selection prior to filtering + * param@[in] coin_selection_params Parameters for the coin selection + * param@[in] allow_mixed_output_types Relax restriction that SelectionResults must be of the same OutputType + * returns If successful, a SelectionResult containing the input set + * If failed, a nullopt + */ +std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, const CoinsResult& available_coins, + const CoinSelectionParams& coin_selection_params, bool allow_mixed_output_types); /** * Attempt to find a valid input set that meets the provided eligibility filter and target. * Multiple coin selection algorithms will be run and the input set that produces the least waste * (according to the waste metric) will be chosen. * - * param@[in] wallet The wallet which provides solving data for the coins - * param@[in] nTargetValue The target value - * param@[in] eligilibity_filter A filter containing rules for which coins are allowed to be included in this selection - * param@[in] coins The vector of coins available for selection prior to filtering - * param@[in] coin_selection_params Parameters for the coin selection - * returns If successful, a SelectionResult containing the input set - * If failed, a nullopt + * param@[in] wallet The wallet which provides solving data for the coins + * param@[in] nTargetValue The target value + * param@[in] eligilibity_filter A filter containing rules for which coins are allowed to be included in this selection + * param@[in] available_coins The struct of coins, organized by OutputType, available for selection prior to filtering + * param@[in] coin_selection_params Parameters for the coin selection + * returns If successful, a SelectionResult containing the input set + * If failed, a nullopt */ -std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, +std::optional<SelectionResult> ChooseSelectionResult(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, const std::vector<COutput>& available_coins, const CoinSelectionParams& coin_selection_params); /** * Select a set of coins such that nTargetValue is met and at least * all coins from coin_control are selected; never select unconfirmed coins if they are not ours * param@[in] wallet The wallet which provides data necessary to spend the selected coins - * param@[in] vAvailableCoins The vector of coins available to be spent + * param@[in] available_coins The struct of coins, organized by OutputType, available for selection prior to filtering * param@[in] nTargetValue The target value * param@[in] coin_selection_params Parameters for this coin selection such as feerates, whether to avoid partial spends, * and whether to subtract the fee from the outputs. * returns If successful, a SelectionResult containing the selected coins * If failed, a nullopt. */ -std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, const CCoinControl& coin_control, +std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const CAmount& nTargetValue, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); struct CreatedTransactionResult diff --git a/src/wallet/test/availablecoins_tests.cpp b/src/wallet/test/availablecoins_tests.cpp new file mode 100644 index 0000000000..01d24da981 --- /dev/null +++ b/src/wallet/test/availablecoins_tests.cpp @@ -0,0 +1,105 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> +#include <wallet/test/wallet_test_fixture.h> + +#include <boost/test/unit_test.hpp> + +namespace wallet { +BOOST_FIXTURE_TEST_SUITE(availablecoins_tests, WalletTestingSetup) +class AvailableCoinsTestingSetup : public TestChain100Setup +{ +public: + AvailableCoinsTestingSetup() + { + CreateAndProcessBlock({}, {}); + wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), m_args, coinbaseKey); + } + + ~AvailableCoinsTestingSetup() + { + wallet.reset(); + } + CWalletTx& AddTx(CRecipient recipient) + { + CTransactionRef tx; + CCoinControl dummy; + { + constexpr int RANDOM_CHANGE_POSITION = -1; + auto res = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, dummy); + BOOST_CHECK(res); + tx = res.GetObj().tx; + } + wallet->CommitTransaction(tx, {}, {}); + CMutableTransaction blocktx; + { + LOCK(wallet->cs_wallet); + blocktx = CMutableTransaction(*wallet->mapWallet.at(tx->GetHash()).tx); + } + CreateAndProcessBlock({CMutableTransaction(blocktx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + + LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, m_node.chainman->ActiveChain().Tip()->GetBlockHash()); + auto it = wallet->mapWallet.find(tx->GetHash()); + BOOST_CHECK(it != wallet->mapWallet.end()); + it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*index=*/1}; + return it->second; + } + + std::unique_ptr<CWallet> wallet; +}; + +BOOST_FIXTURE_TEST_CASE(BasicOutputTypesTest, AvailableCoinsTestingSetup) +{ + CoinsResult available_coins; + BResult<CTxDestination> dest; + LOCK(wallet->cs_wallet); + + // Verify our wallet has one usable coinbase UTXO before starting + // This UTXO is a P2PK, so it should show up in the Other bucket + available_coins = AvailableCoins(*wallet); + BOOST_CHECK_EQUAL(available_coins.size(), 1U); + BOOST_CHECK_EQUAL(available_coins.other.size(), 1U); + + // We will create a self transfer for each of the OutputTypes and + // verify it is put in the correct bucket after running GetAvailablecoins + // + // For each OutputType, We expect 2 UTXOs in our wallet following the self transfer: + // 1. One UTXO as the recipient + // 2. One UTXO from the change, due to payment address matching logic + + // Bech32m + dest = wallet->GetNewDestination(OutputType::BECH32M, ""); + BOOST_ASSERT(dest.HasRes()); + AddTx(CRecipient{{GetScriptForDestination(dest.GetObj())}, 1 * COIN, /*fSubtractFeeFromAmount=*/true}); + available_coins = AvailableCoins(*wallet); + BOOST_CHECK_EQUAL(available_coins.bech32m.size(), 2U); + + // Bech32 + dest = wallet->GetNewDestination(OutputType::BECH32, ""); + BOOST_ASSERT(dest.HasRes()); + AddTx(CRecipient{{GetScriptForDestination(dest.GetObj())}, 2 * COIN, /*fSubtractFeeFromAmount=*/true}); + available_coins = AvailableCoins(*wallet); + BOOST_CHECK_EQUAL(available_coins.bech32.size(), 2U); + + // P2SH-SEGWIT + dest = wallet->GetNewDestination(OutputType::P2SH_SEGWIT, ""); + AddTx(CRecipient{{GetScriptForDestination(dest.GetObj())}, 3 * COIN, /*fSubtractFeeFromAmount=*/true}); + available_coins = AvailableCoins(*wallet); + BOOST_CHECK_EQUAL(available_coins.P2SH_segwit.size(), 2U); + + // Legacy (P2PKH) + dest = wallet->GetNewDestination(OutputType::LEGACY, ""); + BOOST_ASSERT(dest.HasRes()); + AddTx(CRecipient{{GetScriptForDestination(dest.GetObj())}, 4 * COIN, /*fSubtractFeeFromAmount=*/true}); + available_coins = AvailableCoins(*wallet); + BOOST_CHECK_EQUAL(available_coins.legacy.size(), 2U); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace wallet diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index a418105ee1..cd7fd3f4dd 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -67,7 +67,7 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe set.insert(coin); } -static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount& nValue, CFeeRate feerate = CFeeRate(0), int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) +static void add_coin(CoinsResult& available_coins, CWallet& wallet, const CAmount& nValue, CFeeRate feerate = CFeeRate(0), int nAge = 6*24, bool fIsFromMe = false, int nInput =0, bool spendable = false) { CMutableTransaction tx; tx.nLockTime = nextLockTime++; // so all transactions get different hashes @@ -85,7 +85,7 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount assert(ret.second); CWalletTx& wtx = (*ret.first).second; const auto& txout = wtx.tx->vout.at(nInput); - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + available_coins.bech32.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. @@ -129,18 +129,18 @@ static CAmount make_hard_case(int utxos, std::vector<COutput>& utxo_pool) return target; } -inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) +inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& available_coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) { + for (auto& coin : available_coins) { static_groups.emplace_back(); static_groups.back().Insert(coin, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } return static_groups; } -inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& coins, CWallet& wallet, const CoinEligibilityFilter& filter) +inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& available_coins, CWallet& wallet, const CoinEligibilityFilter& filter) { FastRandomContext rand{}; CoinSelectionParams coin_selection_params{ @@ -155,7 +155,7 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput> /*avoid_partial=*/ false, }; static std::vector<OutputGroup> static_groups; - static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /*positive_only=*/false); + static_groups = GroupOutputs(wallet, available_coins, coin_selection_params, filter, /*positive_only=*/false); return static_groups; } @@ -307,18 +307,18 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetupDescriptorScriptPubKeyMans(); - std::vector<COutput> coins; + CoinsResult available_coins; - add_coin(coins, *wallet, 1, coin_selection_params_bnb.m_effective_feerate); - coins.at(0).input_bytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change)); + add_coin(available_coins, *wallet, 1, coin_selection_params_bnb.m_effective_feerate); + available_coins.all().at(0).input_bytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + BOOST_CHECK(!SelectCoinsBnB(GroupCoins(available_coins.all()), 1 * CENT, coin_selection_params_bnb.m_cost_of_change)); // Test fees subtracted from output: - coins.clear(); - add_coin(coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate); - coins.at(0).input_bytes = 40; + available_coins.clear(); + add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate); + available_coins.all().at(0).input_bytes = 40; coin_selection_params_bnb.m_subtract_fee_outputs = true; - const auto result9 = SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); + const auto result9 = SelectCoinsBnB(GroupCoins(available_coins.all()), 1 * CENT, coin_selection_params_bnb.m_cost_of_change); BOOST_CHECK(result9); BOOST_CHECK_EQUAL(result9->GetSelectedValue(), 1 * CENT); } @@ -330,16 +330,16 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetupDescriptorScriptPubKeyMans(); - std::vector<COutput> coins; + CoinsResult available_coins; - add_coin(coins, *wallet, 5 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 3 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 2 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 5 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 3 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 2 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.m_allow_other_inputs = true; - coin_control.Select(coins.at(0).outpoint); + coin_control.Select(available_coins.all().at(0).outpoint); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - const auto result10 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result10 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); } { @@ -349,52 +349,52 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetupDescriptorScriptPubKeyMans(); - std::vector<COutput> coins; + CoinsResult available_coins; // single coin should be selected when effective fee > long term fee coin_selection_params_bnb.m_effective_feerate = CFeeRate(5000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(3000); - add_coin(coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); add_coin(10 * CENT, 2, expected_result); CCoinControl coin_control; - const auto result11 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result11 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result11)); - coins.clear(); + available_coins.clear(); // more coins should be selected when effective fee < long term fee coin_selection_params_bnb.m_effective_feerate = CFeeRate(3000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(5000); - add_coin(coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); add_coin(9 * CENT, 2, expected_result); add_coin(1 * CENT, 2, expected_result); - const auto result12 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result12 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result12)); - coins.clear(); + available_coins.clear(); // pre selected coin should be selected even if disadvantageous coin_selection_params_bnb.m_effective_feerate = CFeeRate(5000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(3000); - add_coin(coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); add_coin(9 * CENT, 2, expected_result); add_coin(1 * CENT, 2, expected_result); coin_control.m_allow_other_inputs = true; - coin_control.Select(coins.at(1).outpoint); // pre select 9 coin - const auto result13 = SelectCoins(*wallet, coins, 10 * CENT, coin_control, coin_selection_params_bnb); + coin_control.Select(available_coins.all().at(1).outpoint); // pre select 9 coin + const auto result13 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result13)); } } @@ -410,175 +410,175 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetupDescriptorScriptPubKeyMans(); - std::vector<COutput> coins; + CoinsResult available_coins; // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) { - coins.clear(); + available_coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 1 * CENT, CENT)); - add_coin(coins, *wallet, 1*CENT, CFeeRate(0), 4); // add a new 1 cent coin + add_coin(available_coins, *wallet, 1*CENT, CFeeRate(0), 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 1 * CENT, CENT)); // but we can find a new 1 cent - const auto result1 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); + const auto result1 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result1); BOOST_CHECK_EQUAL(result1->GetSelectedValue(), 1 * CENT); - add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin + add_coin(available_coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 3 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 3 * CENT, CENT)); // we can make 3 cents of new coins - const auto result2 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 3 * CENT, CENT); + const auto result2 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 3 * CENT, CENT); BOOST_CHECK(result2); BOOST_CHECK_EQUAL(result2->GetSelectedValue(), 3 * CENT); - add_coin(coins, *wallet, 5*CENT); // add a mature 5 cent coin, - add_coin(coins, *wallet, 10*CENT, CFeeRate(0), 3, true); // a new 10 cent coin sent from one of our own addresses - add_coin(coins, *wallet, 20*CENT); // and a mature 20 cent coin + add_coin(available_coins, *wallet, 5*CENT); // add a mature 5 cent coin, + add_coin(available_coins, *wallet, 10*CENT, CFeeRate(0), 3, true); // a new 10 cent coin sent from one of our own addresses + add_coin(available_coins, *wallet, 20*CENT); // and a mature 20 cent coin // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 38 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 38 * CENT, CENT)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), 38 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard_extra), 38 * CENT, CENT)); // but we can make 37 cents if we accept new coins from ourself - const auto result3 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 37 * CENT, CENT); + const auto result3 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 37 * CENT, CENT); BOOST_CHECK(result3); BOOST_CHECK_EQUAL(result3->GetSelectedValue(), 37 * CENT); // and we can make 38 cents if we accept all new coins - const auto result4 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 38 * CENT, CENT); + const auto result4 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 38 * CENT, CENT); BOOST_CHECK(result4); BOOST_CHECK_EQUAL(result4->GetSelectedValue(), 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - const auto result5 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 34 * CENT, CENT); + const auto result5 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 34 * CENT, CENT); BOOST_CHECK(result5); BOOST_CHECK_EQUAL(result5->GetSelectedValue(), 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(result5->GetInputSet().size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - const auto result6 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 7 * CENT, CENT); + const auto result6 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 7 * CENT, CENT); BOOST_CHECK(result6); BOOST_CHECK_EQUAL(result6->GetSelectedValue(), 7 * CENT); BOOST_CHECK_EQUAL(result6->GetInputSet().size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - const auto result7 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 8 * CENT, CENT); + const auto result7 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 8 * CENT, CENT); BOOST_CHECK(result7); BOOST_CHECK(result7->GetSelectedValue() == 8 * CENT); BOOST_CHECK_EQUAL(result7->GetInputSet().size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - const auto result8 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 9 * CENT, CENT); + const auto result8 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 9 * CENT, CENT); BOOST_CHECK(result8); BOOST_CHECK_EQUAL(result8->GetSelectedValue(), 10 * CENT); BOOST_CHECK_EQUAL(result8->GetInputSet().size(), 1U); // now clear out the wallet and start again to test choosing between subsets of smaller coins and the next biggest coin - coins.clear(); + available_coins.clear(); - add_coin(coins, *wallet, 6*CENT); - add_coin(coins, *wallet, 7*CENT); - add_coin(coins, *wallet, 8*CENT); - add_coin(coins, *wallet, 20*CENT); - add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total + add_coin(available_coins, *wallet, 6*CENT); + add_coin(available_coins, *wallet, 7*CENT); + add_coin(available_coins, *wallet, 8*CENT); + add_coin(available_coins, *wallet, 20*CENT); + add_coin(available_coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - const auto result9 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 71 * CENT, CENT); + const auto result9 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 71 * CENT, CENT); BOOST_CHECK(result9); - BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 72 * CENT, CENT)); + BOOST_CHECK(!KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 72 * CENT, CENT)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - const auto result10 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); + const auto result10 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result10); BOOST_CHECK_EQUAL(result10->GetSelectedValue(), 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(result10->GetInputSet().size(), 1U); - add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total + add_coin(available_coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - const auto result11 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); + const auto result11 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result11); BOOST_CHECK_EQUAL(result11->GetSelectedValue(), 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(result11->GetInputSet().size(), 3U); - add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 + add_coin(available_coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - const auto result12 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 16 * CENT, CENT); + const auto result12 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 16 * CENT, CENT); BOOST_CHECK(result12); BOOST_CHECK_EQUAL(result12->GetSelectedValue(), 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(result12->GetInputSet().size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - const auto result13 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 11 * CENT, CENT); + const auto result13 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 11 * CENT, CENT); BOOST_CHECK(result13); BOOST_CHECK_EQUAL(result13->GetSelectedValue(), 11 * CENT); BOOST_CHECK_EQUAL(result13->GetInputSet().size(), 2U); // check that the smallest bigger coin is used - add_coin(coins, *wallet, 1*COIN); - add_coin(coins, *wallet, 2*COIN); - add_coin(coins, *wallet, 3*COIN); - add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - const auto result14 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 95 * CENT, CENT); + add_coin(available_coins, *wallet, 1*COIN); + add_coin(available_coins, *wallet, 2*COIN); + add_coin(available_coins, *wallet, 3*COIN); + add_coin(available_coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents + const auto result14 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 95 * CENT, CENT); BOOST_CHECK(result14); BOOST_CHECK_EQUAL(result14->GetSelectedValue(), 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(result14->GetInputSet().size(), 1U); - const auto result15 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 195 * CENT, CENT); + const auto result15 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 195 * CENT, CENT); BOOST_CHECK(result15); BOOST_CHECK_EQUAL(result15->GetSelectedValue(), 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(result15->GetInputSet().size(), 1U); // empty the wallet and start again, now with fractions of a cent, to test small change avoidance - coins.clear(); - add_coin(coins, *wallet, CENT * 1 / 10); - add_coin(coins, *wallet, CENT * 2 / 10); - add_coin(coins, *wallet, CENT * 3 / 10); - add_coin(coins, *wallet, CENT * 4 / 10); - add_coin(coins, *wallet, CENT * 5 / 10); + available_coins.clear(); + add_coin(available_coins, *wallet, CENT * 1 / 10); + add_coin(available_coins, *wallet, CENT * 2 / 10); + add_coin(available_coins, *wallet, CENT * 3 / 10); + add_coin(available_coins, *wallet, CENT * 4 / 10); + add_coin(available_coins, *wallet, CENT * 5 / 10); // try making 1 * CENT from the 1.5 * CENT // we'll get change smaller than CENT whatever happens, so can expect CENT exactly - const auto result16 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); + const auto result16 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result16); BOOST_CHECK_EQUAL(result16->GetSelectedValue(), CENT); // but if we add a bigger coin, small change is avoided - add_coin(coins, *wallet, 1111*CENT); + add_coin(available_coins, *wallet, 1111*CENT); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - const auto result17 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); + const auto result17 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result17); BOOST_CHECK_EQUAL(result17->GetSelectedValue(), 1 * CENT); // we should get the exact amount // if we add more small coins: - add_coin(coins, *wallet, CENT * 6 / 10); - add_coin(coins, *wallet, CENT * 7 / 10); + add_coin(available_coins, *wallet, CENT * 6 / 10); + add_coin(available_coins, *wallet, CENT * 7 / 10); // and try again to make 1.0 * CENT - const auto result18 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); + const auto result18 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result18); BOOST_CHECK_EQUAL(result18->GetSelectedValue(), 1 * CENT); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change - coins.clear(); + available_coins.clear(); for (int j = 0; j < 20; j++) - add_coin(coins, *wallet, 50000 * COIN); + add_coin(available_coins, *wallet, 50000 * COIN); - const auto result19 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 500000 * COIN, CENT); + const auto result19 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 500000 * COIN, CENT); BOOST_CHECK(result19); BOOST_CHECK_EQUAL(result19->GetSelectedValue(), 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(result19->GetInputSet().size(), 10U); // in ten coins @@ -587,41 +587,41 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: - coins.clear(); - add_coin(coins, *wallet, CENT * 5 / 10); - add_coin(coins, *wallet, CENT * 6 / 10); - add_coin(coins, *wallet, CENT * 7 / 10); - add_coin(coins, *wallet, 1111 * CENT); - const auto result20 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 1 * CENT, CENT); + available_coins.clear(); + add_coin(available_coins, *wallet, CENT * 5 / 10); + add_coin(available_coins, *wallet, CENT * 6 / 10); + add_coin(available_coins, *wallet, CENT * 7 / 10); + add_coin(available_coins, *wallet, 1111 * CENT); + const auto result20 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 1 * CENT, CENT); BOOST_CHECK(result20); BOOST_CHECK_EQUAL(result20->GetSelectedValue(), 1111 * CENT); // we get the bigger coin BOOST_CHECK_EQUAL(result20->GetInputSet().size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) - coins.clear(); - add_coin(coins, *wallet, CENT * 4 / 10); - add_coin(coins, *wallet, CENT * 6 / 10); - add_coin(coins, *wallet, CENT * 8 / 10); - add_coin(coins, *wallet, 1111 * CENT); - const auto result21 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT, CENT); + available_coins.clear(); + add_coin(available_coins, *wallet, CENT * 4 / 10); + add_coin(available_coins, *wallet, CENT * 6 / 10); + add_coin(available_coins, *wallet, CENT * 8 / 10); + add_coin(available_coins, *wallet, 1111 * CENT); + const auto result21 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), CENT, CENT); BOOST_CHECK(result21); BOOST_CHECK_EQUAL(result21->GetSelectedValue(), CENT); // we should get the exact amount BOOST_CHECK_EQUAL(result21->GetInputSet().size(), 2U); // in two coins 0.4+0.6 // test avoiding small change - coins.clear(); - add_coin(coins, *wallet, CENT * 5 / 100); - add_coin(coins, *wallet, CENT * 1); - add_coin(coins, *wallet, CENT * 100); + available_coins.clear(); + add_coin(available_coins, *wallet, CENT * 5 / 100); + add_coin(available_coins, *wallet, CENT * 1); + add_coin(available_coins, *wallet, CENT * 100); // trying to make 100.01 from these three coins - const auto result22 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 10001 / 100, CENT); + const auto result22 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), CENT * 10001 / 100, CENT); BOOST_CHECK(result22); BOOST_CHECK_EQUAL(result22->GetSelectedValue(), CENT * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(result22->GetInputSet().size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - const auto result23 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), CENT * 9990 / 100, CENT); + const auto result23 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), CENT * 9990 / 100, CENT); BOOST_CHECK(result23); BOOST_CHECK_EQUAL(result23->GetSelectedValue(), 101 * CENT); BOOST_CHECK_EQUAL(result23->GetInputSet().size(), 2U); @@ -629,14 +629,14 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // test with many inputs for (CAmount amt=1500; amt < COIN; amt*=10) { - coins.clear(); + available_coins.clear(); // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) for (uint16_t j = 0; j < 676; j++) - add_coin(coins, *wallet, amt); + add_coin(available_coins, *wallet, amt); // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - const auto result24 = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_confirmed), 2000, CENT); + const auto result24 = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_confirmed), 2000, CENT); BOOST_CHECK(result24); if (amt - 2000 < CENT) { @@ -655,17 +655,17 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // test randomness { - coins.clear(); + available_coins.clear(); for (int i2 = 0; i2 < 100; i2++) - add_coin(coins, *wallet, COIN); + add_coin(available_coins, *wallet, COIN); // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - const auto result25 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); + const auto result25 = KnapsackSolver(GroupCoins(available_coins.all()), 50 * COIN, CENT); BOOST_CHECK(result25); - const auto result26 = KnapsackSolver(GroupCoins(coins), 50 * COIN, CENT); + const auto result26 = KnapsackSolver(GroupCoins(available_coins.all()), 50 * COIN, CENT); BOOST_CHECK(result26); BOOST_CHECK(!EqualResult(*result25, *result26)); @@ -676,9 +676,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - const auto result27 = KnapsackSolver(GroupCoins(coins), COIN, CENT); + const auto result27 = KnapsackSolver(GroupCoins(available_coins.all()), COIN, CENT); BOOST_CHECK(result27); - const auto result28 = KnapsackSolver(GroupCoins(coins), COIN, CENT); + const auto result28 = KnapsackSolver(GroupCoins(available_coins.all()), COIN, CENT); BOOST_CHECK(result28); if (EqualResult(*result27, *result28)) fails++; @@ -689,19 +689,19 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // add 75 cents in small change. not enough to make 90 cents, // then try making 90 cents. there are multiple competing "smallest bigger" coins, // one of which should be picked at random - add_coin(coins, *wallet, 5 * CENT); - add_coin(coins, *wallet, 10 * CENT); - add_coin(coins, *wallet, 15 * CENT); - add_coin(coins, *wallet, 20 * CENT); - add_coin(coins, *wallet, 25 * CENT); + add_coin(available_coins, *wallet, 5 * CENT); + add_coin(available_coins, *wallet, 10 * CENT); + add_coin(available_coins, *wallet, 15 * CENT); + add_coin(available_coins, *wallet, 20 * CENT); + add_coin(available_coins, *wallet, 25 * CENT); for (int i = 0; i < RUN_TESTS; i++) { int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - const auto result29 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); + const auto result29 = KnapsackSolver(GroupCoins(available_coins.all()), 90 * CENT, CENT); BOOST_CHECK(result29); - const auto result30 = KnapsackSolver(GroupCoins(coins), 90 * CENT, CENT); + const auto result30 = KnapsackSolver(GroupCoins(available_coins.all()), 90 * CENT, CENT); BOOST_CHECK(result30); if (EqualResult(*result29, *result30)) fails++; @@ -720,14 +720,14 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetupDescriptorScriptPubKeyMans(); - std::vector<COutput> coins; + CoinsResult available_coins; // Test vValue sort order for (int i = 0; i < 1000; i++) - add_coin(coins, *wallet, 1000 * COIN); - add_coin(coins, *wallet, 3 * COIN); + add_coin(available_coins, *wallet, 1000 * COIN); + add_coin(available_coins, *wallet, 3 * COIN); - const auto result = KnapsackSolver(KnapsackGroupOutputs(coins, *wallet, filter_standard), 1003 * COIN, CENT, rand); + const auto result = KnapsackSolver(KnapsackGroupOutputs(available_coins.all(), *wallet, filter_standard), 1003 * COIN, CENT, rand); BOOST_CHECK(result); BOOST_CHECK_EQUAL(result->GetSelectedValue(), 1003 * COIN); BOOST_CHECK_EQUAL(result->GetInputSet().size(), 2U); @@ -750,14 +750,14 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) // Run this test 100 times for (int i = 0; i < 100; ++i) { - std::vector<COutput> coins; + CoinsResult available_coins; CAmount balance{0}; // Make a wallet with 1000 exponentially distributed random inputs for (int j = 0; j < 1000; ++j) { CAmount val = distribution(generator)*10000000; - add_coin(coins, *wallet, val); + add_coin(available_coins, *wallet, val); balance += val; } @@ -780,7 +780,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) /*avoid_partial=*/ false, }; CCoinControl cc; - const auto result = SelectCoins(*wallet, coins, target, cc, cs_params); + const auto result = SelectCoins(*wallet, available_coins, target, cc, cs_params); BOOST_CHECK(result); BOOST_CHECK_GE(result->GetSelectedValue(), target); } diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 816bef6148..5c173773e4 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -137,8 +137,13 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) block.vtx.emplace_back(MakeTransactionRef(tx)); } // Mine block - a.wallet->blockConnected(block, chain.size()); - b.wallet->blockConnected(block, chain.size()); + const uint256& hash = block.GetHash(); + interfaces::BlockInfo info{hash}; + info.prev_hash = &block.hashPrevBlock; + info.height = chain.size(); + info.data = █ + a.wallet->blockConnected(info); + b.wallet->blockConnected(info); // Store the coins for the next block Coins coins_new; for (const auto& tx : block.vtx) { @@ -154,8 +159,13 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) auto& [coins, block]{chain.back()}; if (block.vtx.empty()) return; // Can only disconnect if the block was submitted first // Disconnect block - a.wallet->blockDisconnected(block, chain.size() - 1); - b.wallet->blockDisconnected(block, chain.size() - 1); + const uint256& hash = block.GetHash(); + interfaces::BlockInfo info{hash}; + info.prev_hash = &block.hashPrevBlock; + info.height = chain.size() - 1; + info.data = █ + a.wallet->blockDisconnected(info); + b.wallet->blockDisconnected(info); chain.pop_back(); }); auto& [coins, first_block]{chain.front()}; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 81c01e5023..81183aa738 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -53,9 +53,6 @@ static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } return wallet; } @@ -363,13 +360,13 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 0); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 0); // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx, ISMINE_SPENDABLE), 50*COIN); } static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) @@ -594,7 +591,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) // Lock both coins. Confirm number of available coins drops to 0. { LOCK(wallet->cs_wallet); - BOOST_CHECK_EQUAL(AvailableCoinsListUnspent(*wallet).coins.size(), 2U); + BOOST_CHECK_EQUAL(AvailableCoinsListUnspent(*wallet).size(), 2U); } for (const auto& group : list) { for (const auto& coin : group.second) { @@ -604,7 +601,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) } { LOCK(wallet->cs_wallet); - BOOST_CHECK_EQUAL(AvailableCoinsListUnspent(*wallet).coins.size(), 0U); + BOOST_CHECK_EQUAL(AvailableCoinsListUnspent(*wallet).size(), 0U); } // Confirm ListCoins still returns same result as before, despite coins // being locked. @@ -768,6 +765,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // being blocked wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); + // AddToWallet events for block_tx and mempool_tx BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); @@ -780,6 +778,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // transactionAddedToMempool events are processed promise.set_value(); SyncWithValidationInterfaceQueue(); + // AddToWallet events for block_tx and mempool_tx events are counted a + // second time as the notification queue is processed BOOST_CHECK_EQUAL(addtx_count, 4); @@ -803,7 +803,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 475627c76c..54a3221e2d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1314,30 +1314,31 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe } } -void CWallet::blockConnected(const CBlock& block, int height) +void CWallet::blockConnected(const interfaces::BlockInfo& block) { - const uint256& block_hash = block.GetHash(); + assert(block.data); LOCK(cs_wallet); - m_last_block_processed_height = height; - m_last_block_processed = block_hash; - for (size_t index = 0; index < block.vtx.size(); index++) { - SyncTransaction(block.vtx[index], TxStateConfirmed{block_hash, height, static_cast<int>(index)}); - transactionRemovedFromMempool(block.vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); + m_last_block_processed_height = block.height; + m_last_block_processed = block.hash; + for (size_t index = 0; index < block.data->vtx.size(); index++) { + SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); + transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); } } -void CWallet::blockDisconnected(const CBlock& block, int height) +void CWallet::blockDisconnected(const interfaces::BlockInfo& block) { + assert(block.data); LOCK(cs_wallet); // At block disconnection, this will change an abandoned transaction to // be unconfirmed, whether or not the transaction is added back to the mempool. // User may have to call abandontransaction again. It may be addressed in the // future with a stickier abandoned state or even removing abandontransaction call. - m_last_block_processed_height = height - 1; - m_last_block_processed = block.hashPrevBlock; - for (const CTransactionRef& ptx : block.vtx) { + m_last_block_processed_height = block.height - 1; + m_last_block_processed = *Assert(block.prev_hash); + for (const CTransactionRef& ptx : Assert(block.data)->vtx) { SyncTransaction(ptx, TxStateInactive{}); } } @@ -1678,7 +1679,8 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r /** * Scan the block chain (starting in start_block) for transactions * from or to us. If fUpdate is true, found transactions that already - * exist in the wallet will be updated. + * exist in the wallet will be updated. If max_height is not set, the + * mempool will be scanned as well. * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. @@ -1799,6 +1801,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } } } + if (!max_height) { + WalletLogPrintf("Scanning current mempool transactions.\n"); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); + } ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); @@ -1979,7 +1985,6 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp if (n_signed) { *n_signed = 0; } - const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); LOCK(cs_wallet); // Get all of the previous transactions for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { @@ -2003,6 +2008,8 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp } } + const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); + // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { int n_signed_this_spkm = 0; @@ -3044,20 +3051,31 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf if (tip_height && *tip_height != rescan_height) { - if (chain.havePruned()) { + // Technically we could execute the code below in any case, but performing the + // `while` loop below can make startup very slow, so only check blocks on disk + // if necessary. + if (chain.havePruned() || chain.hasAssumedValidChain()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - // We can't rescan beyond non-pruned blocks, stop and throw an error. + // We can't rescan beyond blocks we don't have data for, stop and throw an error. // This might happen if a user uses an old wallet within a pruned node // or if they ran -disablewallet for a longer time, then decided to re-enable // Exit early and print an error. + // It also may happen if an assumed-valid chain is in use and therefore not + // all block data is available. // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + + error = chain.hasAssumedValidChain() ? + _( + "Assumed-valid: last wallet synchronisation goes beyond " + "available block data. You need to wait for the background " + "validation chain to download more blocks.") : + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return false; } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index ac3e12b51c..abab4a2a9b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -508,8 +508,8 @@ public: CWalletTx* AddToWallet(CTransactionRef tx, const TxState& state, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; - void blockConnected(const CBlock& block, int height) override; - void blockDisconnected(const CBlock& block, int height) override; + void blockConnected(const interfaces::BlockInfo& block) override; + void blockDisconnected(const interfaces::BlockInfo& block) override; void updatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 5e49d0214a..63abf0d9f8 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -95,7 +95,7 @@ class AddrmanTest(BitcoinTestFramework): with open(peers_dat, "wb") as f: f.write(serialize_addrman()[:-1]) self.nodes[0].assert_start_raises_init_error( - expected_msg=init_error("CAutoFile::read: end of file.*"), + expected_msg=init_error("AutoFile::read: end of file.*"), match=ErrorMatch.FULL_REGEX, ) diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index 9d0ad5ef9d..fb4024b1b0 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -82,7 +82,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): msg.hashstop = 0 peer.send_and_ping(msg) time.sleep(5) - assert ("headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0) + assert "headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0 self.log.info("Generating one more block") self.generate(self.nodes[0], 1) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 0e44038196..777f873f70 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -91,7 +91,11 @@ from test_framework.script_util import ( script_to_p2wsh_script, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error, assert_equal +from test_framework.util import ( + assert_raises_rpc_error, + assert_equal, + random_bytes, +) from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, @@ -566,10 +570,6 @@ def random_checksig_style(pubkey): ret = CScript([pubkey, opcode]) return bytes(ret) -def random_bytes(n): - """Return a random bytes object of length n.""" - return bytes(random.getrandbits(8) for i in range(n)) - def bitflipper(expr): """Return a callable that evaluates expr and returns it with a random bitflip.""" def fn(ctx): diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 58f4c91343..b6fa7fbd91 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -141,6 +141,16 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, wallet_watch.getbalance()) + mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') + mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') + + self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC") + assert not os.path.exists(mempooldat1) + result1 = self.nodes[1].savemempool() + assert os.path.isfile(mempooldat1) + assert_equal(result1['filename'], mempooldat1) + os.remove(mempooldat1) + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) @@ -153,8 +163,6 @@ class MempoolPersistTest(BitcoinTestFramework): assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 7) - mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') - mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) result0 = self.nodes[0].savemempool() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 64e66ac30a..3b75b2bc2d 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -122,11 +122,11 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `dummy` txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000' - assert_raises_rpc_error(-1, "JSON value is not a number as expected", self.nodes[0].prioritisetransaction, txid, 'foo', 0) + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type number", self.nodes[0].prioritisetransaction, txid, 'foo', 0) assert_raises_rpc_error(-8, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0.", self.nodes[0].prioritisetransaction, txid, 1, 0) # Test `prioritisetransaction` invalid `fee_delta` - assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type number", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') self.test_diamond() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 193bd3f1cd..d07b144905 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -261,12 +261,12 @@ class BlockchainTest(BitcoinTestFramework): assert_raises_rpc_error(-1, 'getchaintxstats', self.nodes[0].getchaintxstats, 0, '', 0) # Test `getchaintxstats` invalid `nblocks` - assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].getchaintxstats, '') + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type number", self.nodes[0].getchaintxstats, '') assert_raises_rpc_error(-8, "Invalid block count: should be between 0 and the block's height - 1", self.nodes[0].getchaintxstats, -1) assert_raises_rpc_error(-8, "Invalid block count: should be between 0 and the block's height - 1", self.nodes[0].getchaintxstats, self.nodes[0].getblockcount()) # Test `getchaintxstats` invalid `blockhash` - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getchaintxstats, blockhash=0) + assert_raises_rpc_error(-1, "JSON value of type number is not of expected type string", self.nodes[0].getchaintxstats, blockhash=0) assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 1, for '0')", self.nodes[0].getchaintxstats, blockhash='0') assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].getchaintxstats, blockhash='ZZZ0000000000000000000000000000000000000000000000000000000000000') assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getchaintxstats, blockhash='0000000000000000000000000000000000000000000000000000000000000000') @@ -536,7 +536,7 @@ class BlockchainTest(BitcoinTestFramework): datadir = get_datadir_path(self.options.tmpdir, 0) self.log.info("Test getblock with invalid verbosity type returns proper error message") - assert_raises_rpc_error(-1, "JSON value is not an integer as expected", node.getblock, blockhash, "2") + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type number", node.getblock, blockhash, "2") def move_block_file(old, new): old_path = os.path.join(datadir, self.chain, 'blocks', old) diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 948deaaec4..cf9ad3f458 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -301,7 +301,7 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[2].fundrawtransaction, rawtx, {'change_type': None}) + assert_raises_rpc_error(-1, "JSON value of type null is not of expected type string", self.nodes[2].fundrawtransaction, rawtx, {'change_type': None}) assert_raises_rpc_error(-5, "Unknown change type ''", self.nodes[2].fundrawtransaction, rawtx, {'change_type': ''}) rawtx = self.nodes[2].fundrawtransaction(rawtx, {'change_type': 'bech32'}) dec_tx = self.nodes[2].decoderawtransaction(rawtx['hex']) diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 3b6413d4a6..5b7e724728 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -92,7 +92,7 @@ class HelpRpcTest(BitcoinTestFramework): assert_raises_rpc_error(-1, 'help', node.help, 'foo', 'bar') # invalid argument - assert_raises_rpc_error(-1, 'JSON value is not a string as expected', node.help, 0) + assert_raises_rpc_error(-1, "JSON value of type number is not of expected type string", node.help, 0) # help of unknown command assert_equal(node.help('foo'), 'help: unknown command: foo') diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 3e068b94bc..43a73671a5 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -11,10 +11,23 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey, H_POINT from test_framework.messages import ( + COutPoint, + CTransaction, + CTxIn, + CTxOut, MAX_BIP125_RBF_SEQUENCE, WITNESS_SCALE_FACTOR, ser_compact_size, ) +from test_framework.psbt import ( + PSBT, + PSBTMap, + PSBT_GLOBAL_UNSIGNED_TX, + PSBT_IN_RIPEMD160, + PSBT_IN_SHA256, + PSBT_IN_HASH160, + PSBT_IN_HASH256, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -22,6 +35,8 @@ from test_framework.util import ( assert_greater_than, assert_raises_rpc_error, find_output, + find_vout_for_address, + random_bytes, ) from test_framework.wallet_util import bytes_to_wif @@ -764,5 +779,57 @@ class PSBTTest(BitcoinTestFramework): psbt = self.nodes[0].walletprocesspsbt(psbt)["psbt"] self.nodes[0].sendrawtransaction(self.nodes[0].finalizepsbt(psbt)["hex"]) + self.log.info("Test that walletprocesspsbt both updates and signs a non-updated psbt containing Taproot inputs") + addr = self.nodes[0].getnewaddress("", "bech32m") + txid = self.nodes[0].sendtoaddress(addr, 1) + vout = find_vout_for_address(self.nodes[0], txid, addr) + psbt = self.nodes[0].createpsbt([{"txid": txid, "vout": vout}], [{self.nodes[0].getnewaddress(): 0.9999}]) + signed = self.nodes[0].walletprocesspsbt(psbt) + rawtx = self.nodes[0].finalizepsbt(signed["psbt"])["hex"] + self.nodes[0].sendrawtransaction(rawtx) + self.generate(self.nodes[0], 1) + + self.log.info("Test decoding PSBT with per-input preimage types") + # note that the decodepsbt RPC doesn't check whether preimages and hashes match + hash_ripemd160, preimage_ripemd160 = random_bytes(20), random_bytes(50) + hash_sha256, preimage_sha256 = random_bytes(32), random_bytes(50) + hash_hash160, preimage_hash160 = random_bytes(20), random_bytes(50) + hash_hash256, preimage_hash256 = random_bytes(32), random_bytes(50) + + tx = CTransaction() + tx.vin = [CTxIn(outpoint=COutPoint(hash=int('aa' * 32, 16), n=0), scriptSig=b""), + CTxIn(outpoint=COutPoint(hash=int('bb' * 32, 16), n=0), scriptSig=b""), + CTxIn(outpoint=COutPoint(hash=int('cc' * 32, 16), n=0), scriptSig=b""), + CTxIn(outpoint=COutPoint(hash=int('dd' * 32, 16), n=0), scriptSig=b"")] + tx.vout = [CTxOut(nValue=0, scriptPubKey=b"")] + psbt = PSBT() + psbt.g = PSBTMap({PSBT_GLOBAL_UNSIGNED_TX: tx.serialize()}) + psbt.i = [PSBTMap({bytes([PSBT_IN_RIPEMD160]) + hash_ripemd160: preimage_ripemd160}), + PSBTMap({bytes([PSBT_IN_SHA256]) + hash_sha256: preimage_sha256}), + PSBTMap({bytes([PSBT_IN_HASH160]) + hash_hash160: preimage_hash160}), + PSBTMap({bytes([PSBT_IN_HASH256]) + hash_hash256: preimage_hash256})] + psbt.o = [PSBTMap()] + res_inputs = self.nodes[0].decodepsbt(psbt.to_base64())["inputs"] + assert_equal(len(res_inputs), 4) + preimage_keys = ["ripemd160_preimages", "sha256_preimages", "hash160_preimages", "hash256_preimages"] + expected_hashes = [hash_ripemd160, hash_sha256, hash_hash160, hash_hash256] + expected_preimages = [preimage_ripemd160, preimage_sha256, preimage_hash160, preimage_hash256] + for res_input, preimage_key, hash, preimage in zip(res_inputs, preimage_keys, expected_hashes, expected_preimages): + assert preimage_key in res_input + assert_equal(len(res_input[preimage_key]), 1) + assert hash.hex() in res_input[preimage_key] + assert_equal(res_input[preimage_key][hash.hex()], preimage.hex()) + + self.log.info("Test that combining PSBTs with different transactions fails") + tx = CTransaction() + tx.vin = [CTxIn(outpoint=COutPoint(hash=int('aa' * 32, 16), n=0), scriptSig=b"")] + tx.vout = [CTxOut(nValue=0, scriptPubKey=b"")] + psbt1 = PSBT(g=PSBTMap({PSBT_GLOBAL_UNSIGNED_TX: tx.serialize()}), i=[PSBTMap()], o=[PSBTMap()]).to_base64() + tx.vout[0].nValue += 1 # slightly modify tx + psbt2 = PSBT(g=PSBTMap({PSBT_GLOBAL_UNSIGNED_TX: tx.serialize()}), i=[PSBTMap()], o=[PSBTMap()]).to_base64() + assert_raises_rpc_error(-8, "PSBTs not compatible (different transactions)", self.nodes[0].combinepsbt, [psbt1, psbt2]) + assert_equal(self.nodes[0].combinepsbt([psbt1, psbt1]), psbt1) + + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index bbf1f022d7..a858292dd4 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -124,13 +124,13 @@ class RawTransactionsTest(BitcoinTestFramework): # 6. invalid parameters - supply txid and invalid boolean values (strings) for verbose for value in ["True", "False"]: - assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txid=txId, verbose=value) + assert_raises_rpc_error(-1, "not of expected type bool", self.nodes[n].getrawtransaction, txid=txId, verbose=value) # 7. invalid parameters - supply txid and empty array - assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, []) + assert_raises_rpc_error(-1, "not of expected type bool", self.nodes[n].getrawtransaction, txId, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, {}) + assert_raises_rpc_error(-1, "not of expected type bool", self.nodes[n].getrawtransaction, txId, {}) # Make a tx by sending, then generate 2 blocks; block1 has the tx in it tx = self.wallet.send_self_transfer(from_node=self.nodes[2])['txid'] @@ -152,7 +152,7 @@ class RawTransactionsTest(BitcoinTestFramework): # We should not get the tx if we provide an unrelated block assert_raises_rpc_error(-5, "No such transaction found", self.nodes[n].getrawtransaction, txid=tx, blockhash=block2) # An invalid block hash should raise the correct errors - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[n].getrawtransaction, txid=tx, blockhash=True) + assert_raises_rpc_error(-1, "JSON value of type bool is not of expected type string", self.nodes[n].getrawtransaction, txid=tx, blockhash=True) assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 6, for 'foobar')", self.nodes[n].getrawtransaction, txid=tx, blockhash="foobar") assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 8, for 'abcd1234')", self.nodes[n].getrawtransaction, txid=tx, blockhash="abcd1234") foo = "ZZZ0000000000000000000000000000000000000000000000000000000000000" @@ -181,8 +181,8 @@ class RawTransactionsTest(BitcoinTestFramework): # Test `createrawtransaction` invalid `inputs` assert_raises_rpc_error(-3, "Expected type array", self.nodes[0].createrawtransaction, 'foo', {}) - assert_raises_rpc_error(-1, "JSON value is not an object as expected", self.nodes[0].createrawtransaction, ['foo'], {}) - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].createrawtransaction, [{}], {}) + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type object", self.nodes[0].createrawtransaction, ['foo'], {}) + assert_raises_rpc_error(-1, "JSON value of type null is not of expected type string", self.nodes[0].createrawtransaction, [{}], {}) assert_raises_rpc_error(-8, "txid must be of length 64 (not 3, for 'foo')", self.nodes[0].createrawtransaction, [{'txid': 'foo'}], {}) txid = "ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844" assert_raises_rpc_error(-8, f"txid must be hexadecimal string (not '{txid}')", self.nodes[0].createrawtransaction, [{'txid': txid}], {}) @@ -207,7 +207,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Test `createrawtransaction` invalid `outputs` address = getnewdestination()[2] - assert_raises_rpc_error(-1, "JSON value is not an array as expected", self.nodes[0].createrawtransaction, [], 'foo') + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type array", self.nodes[0].createrawtransaction, [], 'foo') self.nodes[0].createrawtransaction(inputs=[], outputs={}) # Should not throw for backwards compatibility self.nodes[0].createrawtransaction(inputs=[], outputs=[]) assert_raises_rpc_error(-8, "Data must be hexadecimal string", self.nodes[0].createrawtransaction, [], {'data': 'foo'}) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index e6d9f9ae3a..4a68312379 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -208,6 +208,20 @@ def tx_from_hex(hex_string): return from_hex(CTransaction(), hex_string) +# like from_hex, but without the hex part +def from_binary(cls, stream): + """deserialize a binary stream (or bytes object) into an object""" + # handle bytes object by turning it into a stream + was_bytes = isinstance(stream, bytes) + if was_bytes: + stream = BytesIO(stream) + obj = cls() + obj.deserialize(stream) + if was_bytes: + assert len(stream.read()) == 0 + return obj + + # Objects that map to bitcoind objects, which can be serialized/deserialized diff --git a/test/functional/test_framework/psbt.py b/test/functional/test_framework/psbt.py new file mode 100644 index 0000000000..68945e7e84 --- /dev/null +++ b/test/functional/test_framework/psbt.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import base64 + +from .messages import ( + CTransaction, + deser_string, + from_binary, + ser_compact_size, +) + + +# global types +PSBT_GLOBAL_UNSIGNED_TX = 0x00 +PSBT_GLOBAL_XPUB = 0x01 +PSBT_GLOBAL_TX_VERSION = 0x02 +PSBT_GLOBAL_FALLBACK_LOCKTIME = 0x03 +PSBT_GLOBAL_INPUT_COUNT = 0x04 +PSBT_GLOBAL_OUTPUT_COUNT = 0x05 +PSBT_GLOBAL_TX_MODIFIABLE = 0x06 +PSBT_GLOBAL_VERSION = 0xfb +PSBT_GLOBAL_PROPRIETARY = 0xfc + +# per-input types +PSBT_IN_NON_WITNESS_UTXO = 0x00 +PSBT_IN_WITNESS_UTXO = 0x01 +PSBT_IN_PARTIAL_SIG = 0x02 +PSBT_IN_SIGHASH_TYPE = 0x03 +PSBT_IN_REDEEM_SCRIPT = 0x04 +PSBT_IN_WITNESS_SCRIPT = 0x05 +PSBT_IN_BIP32_DERIVATION = 0x06 +PSBT_IN_FINAL_SCRIPTSIG = 0x07 +PSBT_IN_FINAL_SCRIPTWITNESS = 0x08 +PSBT_IN_POR_COMMITMENT = 0x09 +PSBT_IN_RIPEMD160 = 0x0a +PSBT_IN_SHA256 = 0x0b +PSBT_IN_HASH160 = 0x0c +PSBT_IN_HASH256 = 0x0d +PSBT_IN_PREVIOUS_TXID = 0x0e +PSBT_IN_OUTPUT_INDEX = 0x0f +PSBT_IN_SEQUENCE = 0x10 +PSBT_IN_REQUIRED_TIME_LOCKTIME = 0x11 +PSBT_IN_REQUIRED_HEIGHT_LOCKTIME = 0x12 +PSBT_IN_TAP_KEY_SIG = 0x13 +PSBT_IN_TAP_SCRIPT_SIG = 0x14 +PSBT_IN_TAP_LEAF_SCRIPT = 0x15 +PSBT_IN_TAP_BIP32_DERIVATION = 0x16 +PSBT_IN_TAP_INTERNAL_KEY = 0x17 +PSBT_IN_TAP_MERKLE_ROOT = 0x18 +PSBT_IN_PROPRIETARY = 0xfc + +# per-output types +PSBT_OUT_REDEEM_SCRIPT = 0x00 +PSBT_OUT_WITNESS_SCRIPT = 0x01 +PSBT_OUT_BIP32_DERIVATION = 0x02 +PSBT_OUT_AMOUNT = 0x03 +PSBT_OUT_SCRIPT = 0x04 +PSBT_OUT_TAP_INTERNAL_KEY = 0x05 +PSBT_OUT_TAP_TREE = 0x06 +PSBT_OUT_TAP_BIP32_DERIVATION = 0x07 +PSBT_OUT_PROPRIETARY = 0xfc + + +class PSBTMap: + """Class for serializing and deserializing PSBT maps""" + + def __init__(self, map=None): + self.map = map if map is not None else {} + + def deserialize(self, f): + m = {} + while True: + k = deser_string(f) + if len(k) == 0: + break + v = deser_string(f) + if len(k) == 1: + k = k[0] + assert k not in m + m[k] = v + self.map = m + + def serialize(self): + m = b"" + for k,v in self.map.items(): + if isinstance(k, int) and 0 <= k and k <= 255: + k = bytes([k]) + m += ser_compact_size(len(k)) + k + m += ser_compact_size(len(v)) + v + m += b"\x00" + return m + +class PSBT: + """Class for serializing and deserializing PSBTs""" + + def __init__(self, *, g=None, i=None, o=None): + self.g = g if g is not None else PSBTMap() + self.i = i if i is not None else [] + self.o = o if o is not None else [] + self.tx = None + + def deserialize(self, f): + assert f.read(5) == b"psbt\xff" + self.g = from_binary(PSBTMap, f) + assert 0 in self.g.map + self.tx = from_binary(CTransaction, self.g.map[0]) + self.i = [from_binary(PSBTMap, f) for _ in self.tx.vin] + self.o = [from_binary(PSBTMap, f) for _ in self.tx.vout] + return self + + def serialize(self): + assert isinstance(self.g, PSBTMap) + assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) + assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) + assert 0 in self.g.map + tx = from_binary(CTransaction, self.g.map[0]) + assert len(tx.vin) == len(self.i) + assert len(tx.vout) == len(self.o) + + psbt = [x.serialize() for x in [self.g] + self.i + self.o] + return b"psbt\xff" + b"".join(psbt) + + def to_base64(self): + return base64.b64encode(self.serialize()).decode("utf8") + + @classmethod + def from_base64(cls, b64psbt): + return from_binary(cls, base64.b64decode(b64psbt)) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 58528b7858..fe61ff95f8 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -12,6 +12,7 @@ import inspect import json import logging import os +import random import re import time import unittest @@ -286,6 +287,13 @@ def sha256sum_file(filename): d = f.read(4096) return h.digest() + +# TODO: Remove and use random.randbytes(n) instead, available in Python 3.9 +def random_bytes(n): + """Return a random bytes object of length n.""" + return bytes(random.getrandbits(8) for i in range(n)) + + # RPC/P2P connection constants and functions ############################################ diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 7f4758af43..e5784eb614 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -156,6 +156,7 @@ BASE_SCRIPTS = [ 'mempool_spend_coinbase.py', 'wallet_avoidreuse.py --legacy-wallet', 'wallet_avoidreuse.py --descriptors', + 'wallet_avoid_mixing_output_types.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', 'p2p_block_sync.py', @@ -205,6 +206,7 @@ BASE_SCRIPTS = [ 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', + 'wallet_miniscript.py', 'feature_maxtipage.py', 'p2p_nobloomfilter_messages.py', 'p2p_filter.py', diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index f7c80f805c..5b836f693f 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -345,31 +345,19 @@ class AddressTypeTest(BitcoinTestFramework): self.log.info("Nodes with addresstype=legacy never use a P2WPKH change output (unless changetype is set otherwise):") self.test_change_output_type(0, [to_address_bech32_1], 'legacy') - if self.options.descriptors: - self.log.info("Nodes with addresstype=p2sh-segwit match the change output") - self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') - self.test_change_output_type(1, [to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32') - else: - self.log.info("Nodes with addresstype=p2sh-segwit match the change output") - self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') - self.test_change_output_type(1, [to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32') + self.log.info("Nodes with addresstype=p2sh-segwit match the change output") + self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') + self.test_change_output_type(1, [to_address_bech32_1], 'bech32') + self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32') + self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32') self.log.info("Nodes with change_type=bech32 always use a P2WPKH change output:") self.test_change_output_type(2, [to_address_bech32_1], 'bech32') self.test_change_output_type(2, [to_address_p2sh], 'bech32') - if self.options.descriptors: - self.log.info("Nodes with addresstype=bech32 match the change output (unless changetype is set otherwise):") - self.test_change_output_type(3, [to_address_bech32_1], 'bech32') - self.test_change_output_type(3, [to_address_p2sh], 'p2sh-segwit') - else: - self.log.info("Nodes with addresstype=bech32 match the change output (unless changetype is set otherwise):") - self.test_change_output_type(3, [to_address_bech32_1], 'bech32') - self.test_change_output_type(3, [to_address_p2sh], 'p2sh-segwit') + self.log.info("Nodes with addresstype=bech32 match the change output (unless changetype is set otherwise):") + self.test_change_output_type(3, [to_address_bech32_1], 'bech32') + self.test_change_output_type(3, [to_address_p2sh], 'p2sh-segwit') self.log.info('getrawchangeaddress defaults to addresstype if -changetype is not set and argument is absent') self.test_address(3, self.nodes[3].getrawchangeaddress(), multisig=False, typ='bech32') diff --git a/test/functional/wallet_avoid_mixing_output_types.py b/test/functional/wallet_avoid_mixing_output_types.py new file mode 100755 index 0000000000..46f41d9c22 --- /dev/null +++ b/test/functional/wallet_avoid_mixing_output_types.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php. +"""Test output type mixing during coin selection + +A wallet may have different types of UTXOs to choose from during coin selection, +where output type is one of the following: + - BECH32M + - BECH32 + - P2SH-SEGWIT + - LEGACY + +This test verifies that mixing different output types is avoided unless +absolutely necessary. Both wallets start with zero funds. Alice mines +enough blocks to have spendable coinbase outputs. Alice sends three +random value payments which sum to 10BTC for each output type to Bob, +for a total of 40BTC in Bob's wallet. + +Bob then sends random valued payments back to Alice, some of which need +unconfirmed change, and we verify that none of these payments contain mixed +inputs. Finally, Bob sends the remainder of his funds, which requires mixing. + +The payment values are random, but chosen such that they sum up to a specified +total. This ensures we are not relying on specific values for the UTXOs, +but still know when to expect mixing due to the wallet being close to empty. + +""" + +import random +from test_framework.test_framework import BitcoinTestFramework +from test_framework.blocktools import COINBASE_MATURITY + +ADDRESS_TYPES = [ + "bech32m", + "bech32", + "p2sh-segwit", + "legacy", +] + + +def is_bech32_address(node, addr): + """Check if an address contains a bech32 output.""" + addr_info = node.getaddressinfo(addr) + return addr_info['desc'].startswith('wpkh(') + + +def is_bech32m_address(node, addr): + """Check if an address contains a bech32m output.""" + addr_info = node.getaddressinfo(addr) + return addr_info['desc'].startswith('tr(') + + +def is_p2sh_segwit_address(node, addr): + """Check if an address contains a P2SH-Segwit output. + Note: this function does not actually determine the type + of P2SH output, but is sufficient for this test in that + we are only generating P2SH-Segwit outputs. + """ + addr_info = node.getaddressinfo(addr) + return addr_info['desc'].startswith('sh(wpkh(') + + +def is_legacy_address(node, addr): + """Check if an address contains a legacy output.""" + addr_info = node.getaddressinfo(addr) + return addr_info['desc'].startswith('pkh(') + + +def is_same_type(node, tx): + """Check that all inputs are of the same OutputType""" + vins = node.getrawtransaction(tx, True)['vin'] + inputs = [] + for vin in vins: + prev_tx, n = vin['txid'], vin['vout'] + inputs.append( + node.getrawtransaction( + prev_tx, + True, + )['vout'][n]['scriptPubKey']['address'] + ) + has_legacy = False + has_p2sh = False + has_bech32 = False + has_bech32m = False + + for addr in inputs: + if is_legacy_address(node, addr): + has_legacy = True + if is_p2sh_segwit_address(node, addr): + has_p2sh = True + if is_bech32_address(node, addr): + has_bech32 = True + if is_bech32m_address(node, addr): + has_bech32m = True + + return (sum([has_legacy, has_p2sh, has_bech32, has_bech32m]) == 1) + + +def generate_payment_values(n, m): + """Return a randomly chosen list of n positive integers summing to m. + Each such list is equally likely to occur.""" + + dividers = sorted(random.sample(range(1, m), n - 1)) + return [a - b for a, b in zip(dividers + [m], [0] + dividers)] + + +class AddressInputTypeGrouping(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [ + [ + "-addresstype=bech32", + "-whitelist=noban@127.0.0.1", + "-txindex", + ], + [ + "-addresstype=p2sh-segwit", + "-whitelist=noban@127.0.0.1", + "-txindex", + ], + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def make_payment(self, A, B, v, addr_type): + fee_rate = random.randint(1, 20) + self.log.debug(f"Making payment of {v} BTC at fee_rate {fee_rate}") + tx = B.sendtoaddress( + address=A.getnewaddress(address_type=addr_type), + amount=v, + fee_rate=fee_rate, + ) + return tx + + def run_test(self): + + # alias self.nodes[i] to A, B for readability + A, B = self.nodes[0], self.nodes[1] + self.generate(A, COINBASE_MATURITY + 5) + + self.log.info("Creating mixed UTXOs in B's wallet") + for v in generate_payment_values(3, 10): + self.log.debug(f"Making payment of {v} BTC to legacy") + A.sendtoaddress(B.getnewaddress(address_type="legacy"), v) + + for v in generate_payment_values(3, 10): + self.log.debug(f"Making payment of {v} BTC to p2sh") + A.sendtoaddress(B.getnewaddress(address_type="p2sh-segwit"), v) + + for v in generate_payment_values(3, 10): + self.log.debug(f"Making payment of {v} BTC to bech32") + A.sendtoaddress(B.getnewaddress(address_type="bech32"), v) + + for v in generate_payment_values(3, 10): + self.log.debug(f"Making payment of {v} BTC to bech32m") + A.sendtoaddress(B.getnewaddress(address_type="bech32m"), v) + + self.generate(A, 1) + + self.log.info("Sending payments from B to A") + for v in generate_payment_values(5, 9): + tx = self.make_payment( + A, B, v, random.choice(ADDRESS_TYPES) + ) + self.generate(A, 1) + assert is_same_type(B, tx) + + tx = self.make_payment(A, B, 30.99, random.choice(ADDRESS_TYPES)) + assert not is_same_type(B, tx) + + +if __name__ == '__main__': + AddressInputTypeGrouping().main() diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0c93821e7f..d49bca6855 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -273,6 +273,26 @@ class WalletTest(BitcoinTestFramework): self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin + if not self.options.descriptors: + self.log.info('Check if mempool is taken into account after import*') + address = self.nodes[0].getnewaddress() + privkey = self.nodes[0].dumpprivkey(address) + self.nodes[0].sendtoaddress(address, 0.1) + self.nodes[0].unloadwallet('') + # check importaddress on fresh wallet + self.nodes[0].createwallet('w1', False, True) + self.nodes[0].importaddress(address) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], 0) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], Decimal('0.1')) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) + assert_equal(self.nodes[0].getbalances()['watchonly']['untrusted_pending'], 0) + self.nodes[0].unloadwallet('w1') + # check importprivkey on fresh wallet + self.nodes[0].createwallet('w2', False, True) + self.nodes[0].importprivkey(privkey) + assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('0.1')) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index f66fab19ac..9cf1b3d2c4 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -414,7 +414,7 @@ class WalletTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string - assert_raises_rpc_error(-1, "not an integer", self.generate, self.nodes[0], "2") + assert_raises_rpc_error(-1, "not of expected type number", self.generate, self.nodes[0], "2") if not self.options.descriptors: diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index ac878ea0aa..686a365584 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -173,8 +173,8 @@ class WalletHDTest(BitcoinTestFramework): # Sethdseed parameter validity assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0) assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif") - assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool") - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True) + assert_raises_rpc_error(-1, "JSON value of type string is not of expected type bool", self.nodes[1].sethdseed, "Not_bool") + assert_raises_rpc_error(-1, "JSON value of type bool is not of expected type string", self.nodes[1].sethdseed, False, True) assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index d9acc8cea5..085ad51c79 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -87,6 +87,7 @@ class Variant(collections.namedtuple("Variant", "call data address_type rescan p assert_equal(len(txs), self.expected_txs) addresses = self.node.listreceivedbyaddress(minconf=0, include_watchonly=True, address_filter=self.address['address']) + if self.expected_txs: assert_equal(len(addresses[0]["txids"]), self.expected_txs) @@ -98,13 +99,18 @@ class Variant(collections.namedtuple("Variant", "call data address_type rescan p assert_equal(tx["category"], "receive") assert_equal(tx["label"], self.label) assert_equal(tx["txid"], txid) - assert_equal(tx["confirmations"], 1 + current_height - confirmation_height) - assert "trusted" not in tx + + # If no confirmation height is given, the tx is still in the + # mempool. + confirmations = (1 + current_height - confirmation_height) if confirmation_height else 0 + assert_equal(tx["confirmations"], confirmations) + if confirmations: + assert "trusted" not in tx address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) assert_equal(address["amount"], self.expected_balance) - assert_equal(address["confirmations"], 1 + current_height - confirmation_height) + assert_equal(address["confirmations"], confirmations) # Verify the transaction is correctly marked watchonly depending on # whether the transaction pays to an imported public key or # imported private key. The test setup ensures that transaction @@ -162,11 +168,12 @@ class ImportRescanTest(BitcoinTestFramework): self.import_deterministic_coinbase_privkeys() self.stop_nodes() - self.start_nodes() + self.start_nodes(extra_args=[["-whitelist=noban@127.0.0.1"]] * self.num_nodes) for i in range(1, self.num_nodes): self.connect_nodes(i, 0) def run_test(self): + # Create one transaction on node 0 with a unique amount for # each possible type of wallet import RPC. for i, variant in enumerate(IMPORT_VARIANTS): @@ -207,7 +214,7 @@ class ImportRescanTest(BitcoinTestFramework): variant.check() # Create new transactions sending to each address. - for i, variant in enumerate(IMPORT_VARIANTS): + for variant in IMPORT_VARIANTS: variant.sent_amount = get_rand_amount() variant.sent_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.sent_amount) self.generate(self.nodes[0], 1) # Generate one block for each send @@ -223,6 +230,46 @@ class ImportRescanTest(BitcoinTestFramework): variant.expected_txs += 1 variant.check(variant.sent_txid, variant.sent_amount, variant.confirmation_height) + self.log.info('Test that the mempool is rescanned as well if the rescan parameter is set to true') + + # The late timestamp and pruned variants are not necessary when testing mempool rescan + mempool_variants = [variant for variant in IMPORT_VARIANTS if variant.rescan != Rescan.late_timestamp and not variant.prune] + # No further blocks are mined so the timestamp will stay the same + timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + + # Create one transaction on node 0 with a unique amount for + # each possible type of wallet import RPC. + for i, variant in enumerate(mempool_variants): + variant.label = "mempool label {} {}".format(i, variant) + variant.address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress( + label=variant.label, + address_type=variant.address_type.value, + )) + variant.key = self.nodes[1].dumpprivkey(variant.address["address"]) + variant.initial_amount = get_rand_amount() + variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) + variant.confirmation_height = 0 + variant.timestamp = timestamp + + assert_equal(len(self.nodes[0].getrawmempool()), len(mempool_variants)) + self.sync_mempools() + + # For each variation of wallet key import, invoke the import RPC and + # check the results from getbalance and listtransactions. + for variant in mempool_variants: + self.log.info('Run import for mempool variant {}'.format(variant)) + expect_rescan = variant.rescan == Rescan.yes + variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] + variant.do_import(variant.timestamp) + if expect_rescan: + variant.expected_balance = variant.initial_amount + variant.expected_txs = 1 + variant.check(variant.initial_txid, variant.initial_amount) + else: + variant.expected_balance = 0 + variant.expected_txs = 0 + variant.check() + if __name__ == "__main__": ImportRescanTest().main() diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index ff11f421a1..525b91a6e0 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -480,7 +480,9 @@ class ImportDescriptorsTest(BitcoinTestFramework): addr = wmulti_pub.getnewaddress('', 'bech32') assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 change_addr = wmulti_pub.getrawchangeaddress('bech32') - assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(change_addr, 'bcrt1qzxl0qz2t88kljdnkzg4n4gapr6kte26390gttrg79x66nt4p04fssj53nl') + assert(send_txid in self.nodes[0].getrawmempool(True)) + assert(send_txid in (x['txid'] for x in wmulti_pub.listunspent(0))) assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) # generate some utxos for next tests diff --git a/test/functional/wallet_miniscript.py b/test/functional/wallet_miniscript.py new file mode 100755 index 0000000000..2252f1e424 --- /dev/null +++ b/test/functional/wallet_miniscript.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test Miniscript descriptors integration in the wallet.""" + +from test_framework.descriptors import descsum_create +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +MINISCRIPTS = [ + # One of two keys + "or_b(pk(tpubD6NzVbkrYhZ4XRMcMFMMFvzVt6jaDAtjZhD7JLwdPdMm9xa76DnxYYP7w9TZGJDVFkek3ArwVsuacheqqPog8TH5iBCX1wuig8PLXim4n9a/*),s:pk(tpubD6NzVbkrYhZ4WsqRzDmkL82SWcu42JzUvKWzrJHQ8EC2vEHRHkXj1De93sD3biLrKd8XGnamXURGjMbYavbszVDXpjXV2cGUERucLJkE6cy/*))", + # A script similar (same spending policy) to BOLT3's offered HTLC (with anchor outputs) + "or_d(pk(tpubD6NzVbkrYhZ4XRMcMFMMFvzVt6jaDAtjZhD7JLwdPdMm9xa76DnxYYP7w9TZGJDVFkek3ArwVsuacheqqPog8TH5iBCX1wuig8PLXim4n9a/*),and_v(and_v(v:pk(tpubD6NzVbkrYhZ4WsqRzDmkL82SWcu42JzUvKWzrJHQ8EC2vEHRHkXj1De93sD3biLrKd8XGnamXURGjMbYavbszVDXpjXV2cGUERucLJkE6cy/*),or_c(pk(tpubD6NzVbkrYhZ4YNwtTWrKRJQzQX3PjPKeUQg1gYh1hiLMkk1cw8SRLgB1yb7JzE8bHKNt6EcZXkJ6AqpCZL1aaRSjnG36mLgbQvJZBNsjWnG/*),v:hash160(7f999c905d5e35cefd0a37673f746eb13fba3640))),older(1)))", + # A Revault Unvault policy with the older() replaced by an after() + "andor(multi(2,tpubD6NzVbkrYhZ4YMQC15JS7QcrsAyfGrGiykweqMmPxTkEVScu7vCZLNpPXW1XphHwzsgmqdHWDQAfucbM72EEB1ZEyfgZxYvkZjYVXx1xS9p/*,tpubD6NzVbkrYhZ4WkCyc7E3z6g6NkypHMiecnwc4DpWHTPqFdteRGkEKukdrSSyJGNnGrHNMfy4BCw2UXo5soYRCtCDDfy4q8pc8oyB7RgTFv8/*),and_v(v:multi(4,030f64b922aee2fd597f104bc6cb3b670f1ca2c6c49b1071a1a6c010575d94fe5a,02abe475b199ec3d62fa576faee16a334fdb86ffb26dce75becebaaedf328ac3fe,0314f3dc33595b0d016bb522f6fe3a67680723d842c1b9b8ae6b59fdd8ab5cccb4,025eba3305bd3c829e4e1551aac7358e4178832c739e4fc4729effe428de0398ab),after(424242)),thresh(4,pkh(tpubD6NzVbkrYhZ4YVrNggiT2ptVHwnFbLBqDkCtV5HkxR4WtcRLAQReKTkqZGNcV6GE7cQsmpBzzSzhk16DUwB1gn1L7ZPnJF2dnNePP1uMBCY/*),a:pkh(tpubD6NzVbkrYhZ4YU9vM1s53UhD75UyJatx8EMzMZ3VUjR2FciNfLLkAw6a4pWACChzobTseNqdWk4G7ZdBqRDLtLSACKykTScmqibb1ZrCvJu/*),a:pkh(tpubD6NzVbkrYhZ4YUHcFfuH9iEBLiH8CBRJTpS7X3qjHmh82m1KCNbzs6w9gyK8oWHSZmKHWcakAXCGfbKg6xoCvKzQCWAHyxaC7QcWfmzyBf4/*),a:pkh(tpubD6NzVbkrYhZ4XXEmQtS3sgxpJbMyMg4McqRR1Af6ULzyrTRnhwjyr1etPD7svap9oFtJf4MM72brUb5o7uvF2Jyszc5c1t836fJW7SX2e8D/*)))", + # Liquid-like federated pegin with emergency recovery keys + "or_i(and_b(pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),a:and_b(pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),a:and_b(pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),a:and_b(pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),s:pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2))))),and_v(v:thresh(2,pkh(tpubD6NzVbkrYhZ4YK67cd5fDe4fBVmGB2waTDrAt1q4ey9HPq9veHjWkw3VpbaCHCcWozjkhgAkWpFrxuPMUrmXVrLHMfEJ9auoZA6AS1g3grC/*),a:pkh(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),a:pkh(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068)),older(4209713)))", +] + + +class WalletMiniscriptTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + def watchonly_test(self, ms): + self.log.info(f"Importing Miniscript '{ms}'") + desc = descsum_create(f"wsh({ms})") + assert self.ms_wo_wallet.importdescriptors( + [ + { + "desc": desc, + "active": True, + "range": 2, + "next_index": 0, + "timestamp": "now", + } + ] + )[0]["success"] + + self.log.info("Testing we derive new addresses for it") + assert_equal( + self.ms_wo_wallet.getnewaddress(), self.funder.deriveaddresses(desc, 0)[0] + ) + assert_equal( + self.ms_wo_wallet.getnewaddress(), self.funder.deriveaddresses(desc, 1)[1] + ) + + self.log.info("Testing we detect funds sent to one of them") + addr = self.ms_wo_wallet.getnewaddress() + txid = self.funder.sendtoaddress(addr, 0.01) + self.wait_until( + lambda: len(self.ms_wo_wallet.listunspent(minconf=0, addresses=[addr])) == 1 + ) + utxo = self.ms_wo_wallet.listunspent(minconf=0, addresses=[addr])[0] + assert utxo["txid"] == txid and not utxo["solvable"] # No satisfaction logic (yet) + + def run_test(self): + self.log.info("Making a descriptor wallet") + self.funder = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet( + wallet_name="ms_wo", descriptors=True, disable_private_keys=True + ) + self.ms_wo_wallet = self.nodes[0].get_wallet_rpc("ms_wo") + + # Sanity check we wouldn't let an insane Miniscript descriptor in + res = self.ms_wo_wallet.importdescriptors( + [ + { + "desc": descsum_create( + "wsh(and_b(ripemd160(1fd9b55a054a2b3f658d97e6b84cf3ee00be429a),a:1))" + ), + "active": False, + "timestamp": "now", + } + ] + )[0] + assert not res["success"] + assert "is not sane: witnesses without signature exist" in res["error"]["message"] + + # Test we can track any type of Miniscript + for ms in MINISCRIPTS: + self.watchonly_test(ms) + + +if __name__ == "__main__": + WalletMiniscriptTest().main() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index dcb82bbbe9..99e472a7b4 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -356,7 +356,7 @@ class MultiWalletTest(BitcoinTestFramework): self.log.info("Test dynamic wallet unloading") # Test `unloadwallet` errors - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet) + assert_raises_rpc_error(-1, "JSON value of type null is not of expected type string", self.nodes[0].unloadwallet) assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy") assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet) assert_raises_rpc_error(-8, "RPC endpoint wallet and wallet_name parameter specify different wallets", w1.unloadwallet, "w2"), diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index 5d157eb4b1..a0f17ac119 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -22,6 +22,10 @@ EXPECTED_CIRCULAR_DEPENDENCIES = ( "wallet/fees -> wallet/wallet -> wallet/fees", "wallet/wallet -> wallet/walletdb -> wallet/wallet", "kernel/coinstats -> validation -> kernel/coinstats", + "kernel/mempool_persist -> validation -> kernel/mempool_persist", + + # Temporary, removed in followup https://github.com/bitcoin/bitcoin/pull/24230 + "index/base -> node/context -> net_processing -> index/blockfilterindex -> index/base", ) CODE_DIR = "src" |