diff options
135 files changed, 3028 insertions, 1163 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 3f52adde8a..e5a2f36f4d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -111,7 +111,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64, unit tests, no gui tests, no boost::process, no functional tests' + name: 'Win64, unit tests, no gui tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE persistent_worker: labels: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1e171c0e3..1cd40be541 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: name: 'macOS 13 native, x86_64, no depends, sqlite only, gui' # Use latest image, but hardcode version to avoid silent upgrades (and breaks). # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-13 # Use M1 once available https://github.com/github/roadmap/issues/528 + runs-on: macos-13 # No need to run on the read-only mirror, unless it is a PR. if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' @@ -127,8 +127,8 @@ jobs: CCACHE_MAXSIZE: '200M' CI_CCACHE_VERSION: '4.7.5' CI_QT_CONF: '-release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' - CI_QT_DIR: 'qt-everywhere-src-5.15.10' - CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.10/single/qt-everywhere-opensource-src-5.15.10.zip' + CI_QT_DIR: 'qt-everywhere-src-5.15.11' + CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip' PYTHONUTF8: 1 TEST_RUNNER_TIMEOUT_FACTOR: 40 @@ -136,37 +136,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Fix Visual Studio installation - # See: https://github.com/actions/runner-images/issues/7832#issuecomment-1617585694. - run: | - Set-Location "C:\Program Files (x86)\Microsoft Visual Studio\Installer\" - $InstallPath = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise" - $componentsToRemove= @( - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM.Spectre" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64" - "Microsoft.VisualStudio.Component.VC.14.35.17.5.MFC.ARM64.Spectre" - ) - [string]$workloadArgs = $componentsToRemove | ForEach-Object {" --remove " + $_} - $Arguments = ('/c', "vs_installer.exe", 'modify', '--installPath', "`"$InstallPath`"",$workloadArgs, '--quiet', '--norestart', '--nocache') - # should be run twice - $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden - $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden - - name: Configure Developer Command Prompt for Microsoft Visual C++ # Using microsoft/setup-msbuild is not enough. uses: ilammy/msvc-dev-cmd@v1 diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 602b57fe43..5e65257d77 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -4,8 +4,10 @@ dnl permitted in any medium without royalty provided the copyright notice dnl and this notice are preserved. This file is offered as-is, without any dnl warranty. -# Some versions of gcc/libstdc++ require linking with -latomic if -# using the C++ atomic library. +# Clang prior to version 15, when building for 32-bit, +# and linking against libstdc++, requires linking with +# -latomic if using the C++ atomic library. +# Can be tested with: clang++ test.cpp -m32 # # Sourced from http://bugs.debian.org/797228 diff --git a/build_msvc/README.md b/build_msvc/README.md index 8206620c3b..cc2cd91e13 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -32,7 +32,7 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.10.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.10/single/qt-everywhere-opensource-src-5.15.10.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.11.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands: ```cmd diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index 20cdb7bb6e..fe01da28c8 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -58,7 +58,7 @@ <Link> <SubSystem>Windows</SubSystem> <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions> + <AdditionalOptions>/LTCG:OFF</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> @@ -72,7 +72,6 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> diff --git a/build_msvc/common.init.vcxproj.in b/build_msvc/common.init.vcxproj.in index 9cce546268..d54e559c9f 100644 --- a/build_msvc/common.init.vcxproj.in +++ b/build_msvc/common.init.vcxproj.in @@ -90,7 +90,7 @@ <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4244;4267;4715;4805</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> @@ -98,9 +98,6 @@ <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> <RandomizedBaseAddress>true</RandomizedBaseAddress> </Link> - <Lib> - <AdditionalOptions>/ignore:4221</AdditionalOptions> - </Lib> </ItemDefinitionGroup> <Import Project="common.init.vcxproj.user" Condition="Exists('common.init.vcxproj.user')" /> </Project> diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj index b34593fe5c..60e57caa57 100644 --- a/build_msvc/libminisketch/libminisketch.vcxproj +++ b/build_msvc/libminisketch/libminisketch.vcxproj @@ -28,7 +28,7 @@ </ItemGroup> <ItemDefinitionGroup> <ClCompile> - <DisableSpecificWarnings>4060;4065;4146;4244;4267;4554</DisableSpecificWarnings> + <DisableSpecificWarnings>4060;4065;4146;4244;4267</DisableSpecificWarnings> <PreprocessorDefinitions>HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> </ItemDefinitionGroup> diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 777515aa3a..7ea4b96534 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -17,7 +17,7 @@ <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;ENABLE_MODULE_ELLSWIFT;%(PreprocessorDefinitions)</PreprocessorDefinitions> <UndefinePreprocessorDefinitions>USE_ASM_X86_64;%(UndefinePreprocessorDefinitions)</UndefinePreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings> + <DisableSpecificWarnings>4146;4244;4267</DisableSpecificWarnings> </ClCompile> </ItemDefinitionGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 3776317fc7..c5a32a9711 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -82,7 +82,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions> + <AdditionalOptions>/LTCG:OFF</AdditionalOptions> </Link> </ItemDefinitionGroup> @@ -92,7 +92,6 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> <ItemGroup> diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 2cecf8510e..1befbc6164 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,3 +15,4 @@ export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ LDFLAGS='--rtlib=compiler-rt -lgcc_s' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" +export NO_WERROR=1 # Temporary workaround to avoid -Wdeprecated-declarations from KJ diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 122f044b58..3585b2b417 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -18,3 +18,4 @@ export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" export CCACHE_MAXSIZE=200M +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-17" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index a0b054ab40..b15df4b6cc 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -67,8 +67,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then fi if [[ "${RUN_TIDY}" == "true" ]]; then - ${CI_RETRY_EXE} git clone https://github.com/include-what-you-use/include-what-you-use -b master /include-what-you-use - git -C /include-what-you-use checkout a138eaac254e5a472464e31d5ec418fe6e6f1fc7 + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi diff --git a/configure.ac b/configure.ac index 6dfd2abfb2..6add570d00 100644 --- a/configure.ac +++ b/configure.ac @@ -1424,15 +1424,13 @@ dnl Check for libminiupnpc (optional) if test "$use_upnp" != "no"; then TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS" - AC_CHECK_HEADERS( - [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], - [have_miniupnpc=no] - ) + AC_CHECK_HEADERS([miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [], [have_miniupnpc=no]) - dnl The minimum supported miniUPnPc API version is set to 17. This excludes - dnl versions with known vulnerabilities. if test "$have_miniupnpc" != "no"; then + AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS]) + + dnl The minimum supported miniUPnPc API version is set to 17. This excludes + dnl versions with known vulnerabilities. AC_MSG_CHECKING([whether miniUPnPc API version is supported]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include <miniupnpc/miniupnpc.h> @@ -1457,9 +1455,12 @@ dnl Check for libnatpmp (optional). if test "$use_natpmp" != "no"; then TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS" - AC_CHECK_HEADERS([natpmp.h], - [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])], - [have_natpmp=no]) + AC_CHECK_HEADERS([natpmp.h], [], [have_natpmp=no]) + + if test "$have_natpmp" != "no"; then + AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS]) + fi + CPPFLAGS="$TEMP_CPPFLAGS" fi diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 0b5f77d01e..54edfecb26 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -86,7 +86,11 @@ mkdir -p "$DISTSRC" signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist # Make a .zip from dist/ - zip "${OUTDIR}/${DISTNAME}-${HOST}.zip" dist/* + cd dist/ + find . -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find . | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST}.zip" ;; *) exit 1 diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 3c4ad6cdbc..d7cab862e4 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -11,7 +11,7 @@ (gnu packages gawk) (gnu packages gcc) ((gnu packages installers) #:select (nsis-x86_64)) - ((gnu packages linux) #:select (linux-libre-headers-5.15 util-linux)) + ((gnu packages linux) #:select (linux-libre-headers-6.1 util-linux)) (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) @@ -19,7 +19,6 @@ ((gnu packages python) #:select (python-minimal)) ((gnu packages python-build) #:select (python-tomli)) ((gnu packages python-crypto) #:select (python-asn1crypto)) - ((gnu packages python-web) #:select (python-requests)) ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) (guix build-system cmake) @@ -93,7 +92,7 @@ chain for " target " development.")) (license (package-license xgcc))))) (define base-gcc gcc-10) -(define base-linux-kernel-headers linux-libre-headers-5.15) +(define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target #:key @@ -445,7 +444,7 @@ and endian independent.") (license license:expat))) (define-public python-signapple - (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) + (let ((commit "7a96b4171a360abf0f0f56e499f8f9ed2116280d")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -458,14 +457,13 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) + "0aa4k180jnpal15yhncnm3g3z9gzmi7qb25q5l0kaj444a1p2pm4")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) ("python-oscrypto" ,python-oscrypto) ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) - ("python-requests" ,python-requests) ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index f4778c1ecd..47df202771 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -8,6 +8,7 @@ $(package)_dependencies=native_$(package) define $(package)_set_vars := $(package)_config_opts := --with-external-capnp +$(package)_config_opts += --without-openssl $(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" $(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" $(package)_config_opts_android := --disable-shared diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk index ed5a6deee2..ad87eed354 100644 --- a/depends/packages/native_capnp.mk +++ b/depends/packages/native_capnp.mk @@ -1,9 +1,13 @@ package=native_capnp -$(package)_version=0.7.0 +$(package)_version=1.0.1 $(package)_download_path=https://capnproto.org/ $(package)_download_file=capnproto-c++-$($(package)_version).tar.gz $(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz -$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41 +$(package)_sha256_hash=0f7f4b8a76a2cdb284fddef20de8306450df6dd031a47a15ac95bc43c3358e09 + +define $(package)_set_vars + $(package)_config_opts = --without-openssl +endef define $(package)_config_cmds $($(package)_autoconf) diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index e647afba5f..c91ffb0334 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=1af83d15239ccfa7e47b8764029320953dd7fdf1 +$(package)_version=61d5a0e661f20a4928fbf868ec9c3c6f17883cc7 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=e5587d3feedc7f8473f178a89b94163a11076629825d664964799bbbd5844da5 +$(package)_sha256_hash=5cfda224cc2ce913f2493f843317e0fca3184f6d7c1434c9754b2e7dca440ab5 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 047d1d5aee..b6d8864383 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ package=qt -$(package)_version=5.15.10 +$(package)_version=5.15.11 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=c0d06cb18d20f10bf7ad53552099e097ec39362d30a5d6f104724f55fa1c8fb9 +$(package)_sha256_hash=425ad301acd91ca66c10c0dabee0704e2d0cd2801a6b670115800cbb95f84846 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert @@ -25,10 +25,10 @@ $(package)_patches += memory_resource.patch $(package)_patches += windows_lto.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=38b942bc7e62794dd072945c8a92bb9dfffed24070aea300327a3bb42f855609 +$(package)_qttranslations_sha256_hash=a31785948c640b7c66d9fe2db4993728ca07f64e41c560b3625ad191b276ff20 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=66f46c9729c831dce431778a9c561cca32daceaede1c7e58568d7a5898167dae +$(package)_qttools_sha256_hash=7cd847ae6ff09416df617136eadcaf0eb98e3bc9b89979219a3ea8111fb8d339 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -134,9 +134,6 @@ $(package)_config_opts_darwin += -no-feature-corewlan $(package)_config_opts_darwin += -no-freetype $(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) -# Optimizing using > -O1 causes non-determinism when building across arches. -$(package)_config_opts_aarch64_darwin += "QMAKE_CFLAGS_OPTIMIZE_FULL = -O1" - ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 7dbd68fe9c..89c96026fb 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -980,6 +980,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) +@@ -979,6 +979,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/doc/README.md b/doc/README.md index c570432aa4..446684b482 100644 --- a/doc/README.md +++ b/doc/README.md @@ -79,6 +79,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Init Scripts (systemd/upstart/openrc)](init.md) - [Managing Wallets](managing-wallets.md) - [Multisig Tutorial](multisig-tutorial.md) +- [Offline Signing Tutorial](offline-signing-tutorial.md) - [P2P bad ports definition and list](p2p-bad-ports.md) - [PSBT support](psbt.md) - [Reduce Memory](reduce-memory.md) diff --git a/doc/dependencies.md b/doc/dependencies.md index 9d9380bf79..41079b93ab 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.10](https://github.com/bitcoin/bitcoin/pull/28561) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.11](https://github.com/bitcoin/bitcoin/pull/28769) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/offline-signing-tutorial.md b/doc/offline-signing-tutorial.md new file mode 100644 index 0000000000..4761cf256b --- /dev/null +++ b/doc/offline-signing-tutorial.md @@ -0,0 +1,255 @@ +# Offline Signing Tutorial + +This tutorial will describe how to use two instances of Bitcoin Core, one online and one offline, to greatly increase security by not having private keys reside on a networked device. + +Maintaining an air-gap between private keys and any network connections drastically reduces the opportunity for those keys to be exfiltrated from the user. + +This workflow uses [Partially Signed Bitcoin Transactions](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) (PSBTs) to transfer the transaction to and from the offline wallet for signing using the private keys. + +> [!NOTE] +> While this tutorial demonstrates the process using `signet` network, you should omit the `-signet` flag in the provided commands when working with `mainnet`. + +## Overview +In this tutorial we have two hosts, both running Bitcoin v25.0 + +* `offline` host which is disconnected from all networks (internet, Tor, wifi, bluetooth etc.) and does not have, or need, a copy of the blockchain. +* `online` host which is a regular online node with a synced blockchain. + +We are going to first create an `offline_wallet` on the offline host. We will then create a `watch_only_wallet` on the online host using public key descriptors exported from the `offline_wallet`. Next we will receive some coins into the wallet. In order to spend these coins we'll create an unsigned PSBT using the `watch_only_wallet`, sign the PSBT using the private keys in the `offline_wallet`, and finally broadcast the signed PSBT using the online host. + +### Requirements +- [jq](https://jqlang.github.io/jq/) installation - This tutorial uses jq to process certain fields from JSON RPC responses, but this convenience is optional. + +### Create and Prepare the `offline_wallet` + +1. On the offline machine create a wallet named `offline_wallet` secured by a wallet `passphrase`. This wallet will contain private keys and must remain unconnected to any networks at all times. + +```sh +[offline]$ ./src/bitcoin-cli -signet -named createwallet \ + wallet_name="offline_wallet" \ + passphrase="** enter passphrase **" + +{ + "name": "offline_wallet" +} +``` + +> [!NOTE] +> The use of a passphrase is crucial to encrypt the wallet.dat file. This encryption ensures that even if an unauthorized individual gains access to the offline host, they won't be able to access the wallet's contents. Further details about securing your wallet can be found in [Managing the Wallet](https://github.com/bitcoin/bitcoin/blob/master/doc/managing-wallets.md#12-encrypting-the-wallet) + +2. Export the public key-only descriptors from the offline host to a JSON file named `descriptors.json`. We use `jq` here to extract the `.descriptors` field from the full RPC response. + +```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" listdescriptors \ + | jq -r '.descriptors' \ + >> /path/to/descriptors.json +``` + +> [!NOTE] +> The `descriptors.json` file will be transferred to the online machine (e.g. using a USB flash drive) where it can be imported to create a related watch-only wallet. + +### Create the online `watch_only_wallet` + +1. On the online machine create a blank watch-only wallet which has private keys disabled and is named `watch_only_wallet`. This is achieved by using the `createwallet` options: `disable_private_keys=true, blank=true`. + +The `watch_only_wallet` wallet will be used to track and validate incoming transactions, create unsigned PSBTs when spending coins, and broadcast signed and finalized PSBTs. + +> [!NOTE] +> `disable_private_keys` indicates that the wallet should refuse to import private keys, i.e. will be a dedicated watch-only wallet. + +```sh +[online]$ ./src/bitcoin-cli -signet -named createwallet \ + wallet_name="watch_only_wallet" \ + disable_private_keys=true + +{ + "name": "watch_only_wallet" +} +``` + +2. Import the `offline_wallet`s public key descriptors to the online `watch_only_wallet` using the `descriptors.json` file created on the offline wallet. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" importdescriptors "$(cat /path/to/descriptors.json)" + +[ + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + }, + { + "success": true + } +] +``` +> [!NOTE] +> Multiple success values indicate that multiple descriptors, for different address types, have been successfully imported. This allows generating different address types on the `watch_only_wallet`. + +### Fund the `offline_wallet` + +At this point, it's important to understand that both the `offline_wallet` and online `watch_only_wallet` share the same public keys. As a result, they generate the same addresses. Transactions can be created using either wallet, but valid signatures can only be added by the `offline_wallet` as only it has the private keys. + +1. Generate an address to receive coins. You can use _either_ the `offline_wallet` or the online `watch_only_wallet` to generate this address, as they will produce the same addresses. For the sake of this guide, we'll use the online `watch_only_wallet` to generate the address. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" getnewaddress + +tb1qtu5qgc6ddhmqm5yqjvhg83qgk2t4ewajg0h6yh +``` + +2. Visit a faucet like https://signet.bc-2.jp and enter your address from the previous command to receive a small amount of signet coins to this address. + +3. Confirm that coins were received using the online `watch_only_wallet`. Note that the transaction may take a few moments before being received on your local node, depending on its connectivity. Just re-run the command periodically until the transaction is received. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" listunspent + +[ + { + "txid": "0f3953dfc3eb8e753cd1633151837c5b9953992914ff32b7de08c47f1f29c762", + "vout": 1, + "address": "tb1qtu5qgc6ddhmqm5yqjvhg83qgk2t4ewajg0h6yh", + "label": "", + "scriptPubKey": "00145f2804634d6df60dd080932e83c408b2975cbbb2", + "amount": 0.01000000, + "confirmations": 4, + "spendable": true, + "solvable": true, + "desc": "wpkh([306c734f/84h/1h/0h/0/0]025932ccee7590158f7e08bb36290d135d30a0b045163da896e1cd7645ec4223a9)#xytvyr4a", + "parent_descs": [ + "wpkh([306c734f/84h/1h/0h]tpubDCJnY92ib4Zu3qd6wrBXEjG436tQdA2tDiJU2iSJYjkNS1darssPWKaBfojhjUF5vMLBcxbN2r93pmFMz2zyTEZuNx9JDo9rWqoHhATW3Uz/0/*)#7mh08dkg" + ], + "safe": true + } +] +``` + +### Create and Export an Unsigned PSBT + +1. Get a destination address for the transaction. In this tutorial we'll be sending funds to the address `tb1q9k5w0nhnhyeh78snpxh0t5t7c3lxdeg3erez32`, but if you don't need the coins for further testing you could send the coins back to the faucet. + +2. Create a funded but unsigned PSBT to the destination address with the online `watch_only_wallet` by using `send [{"address":amount},...]` and export the unsigned PSBT to a file `funded_psbt.txt` for easy portability to the `offline_wallet` for signing: + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" send \ + '{"tb1q9k5w0nhnhyeh78snpxh0t5t7c3lxdeg3erez32": 0.009}' \ + | jq -r '.psbt' \ + >> /path/to/funded_psbt.txt + +[online]$ cat /path/to/funded_psbt.txt + +cHNidP8BAHECAAAAAWLHKR9/xAjetzL/FCmZU5lbfINRMWPRPHWO68PfUzkPAQAAAAD9////AoA4AQAAAAAAFgAULajnzvO5M38eEwmu9dF+xH5m5RGs0g0AAAAAABYAFMaT0f/Wp2DCZzL6dkJ3GhWj4Y9vAAAAAAABAHECAAAAAY+dRPEBrGopkw4ugSzS9npzJDEIrE/bq1XXI0KbYnYrAQAAAAD+////ArKaXgAAAAAAFgAUwEc4LdoxSjbWo/2Ue+HS+QjwfiBAQg8AAAAAABYAFF8oBGNNbfYN0ICTLoPECLKXXLuyYW8CAAEBH0BCDwAAAAAAFgAUXygEY01t9g3QgJMug8QIspdcu7IiBgJZMszudZAVj34IuzYpDRNdMKCwRRY9qJbhzXZF7EIjqRgwbHNPVAAAgAEAAIAAAACAAAAAAAAAAAAAACICA7BlBnyAR4F2UkKuSX9MFhYCsn6j//z9i7lHDm1O0CU0GDBsc09UAACAAQAAgAAAAIABAAAAAAAAAAA= +``` +> [!NOTE] +> Leaving the `input` array empty in the above `walletcreatefundedpsbt` command is permitted and will cause the wallet to automatically select appropriate inputs for the transaction. + +### Decode and Analyze the Unsigned PSBT + +Decode and analyze the unsigned PSBT on the `offline_wallet` using the `funded_psbt.txt` file: + +```sh +[offline]$ ./src/bitcoin-cli -signet decodepsbt $(cat /path/to/funded_psbt.txt) + +{ + ... +} + +[offline]$ ./src/bitcoin-cli -signet analyzepsbt $(cat /path/to/funded_psbt.txt) + +{ + "inputs": [ + { + "has_utxo": true, + "is_final": false, + "next": "signer", + "missing": { + "signatures": [ + "5f2804634d6df60dd080932e83c408b2975cbbb2" + ] + } + } + ], + "estimated_vsize": 141, + "estimated_feerate": 0.00100000, + "fee": 0.00014100, + "next": "signer" +} +``` + +Notice that the analysis of the PSBT shows that "signatures" are missing and should be provided by the private key corresponding to the public key hash (hash160) "5f2804634d6df60dd080932e83c408b2975cbbb2" + +### Process and Sign the PSBT + +1. Unlock the `offline_wallet` with the Passphrase: + +Use the walletpassphrase command to unlock the `offline_wallet` with the passphrase. You should specify the passphrase and a timeout (in seconds) for how long you want the wallet to remain unlocked. + +```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" walletpassphrase "** enter passphrase **" 60 +``` + +2. Process, sign and finalize the PSBT on the `offline_wallet` using the `walletprocesspsbt` command, saving the output to a file `final_psbt.txt`. + + ```sh +[offline]$ ./src/bitcoin-cli -signet -rpcwallet="offline_wallet" walletprocesspsbt \ + $(cat /path/to/funded_psbt.txt) \ + | jq -r .hex \ + >> /path/to/final_psbt.txt + ``` + +### Broadcast the Signed and Finalized PSBT +Broadcast the funded, signed and finalized PSBT `final_psbt.txt` using `sendrawtransaction` with an online node: + +```sh +[online]$ ./src/bitcoin-cli -signet sendrawtransaction $(cat /path/to/final_psbt.txt) + +c2430a0e46df472b04b0ca887bbcd5c4abf7b2ce2eb71de981444a80e2b96d52 +``` + +### Confirm Wallet Balance + +Confirm the updated balance of the offline wallet using the `watch_only_wallet`. + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" getbalances + +{ + "mine": { + "trusted": 0.00085900, + "untrusted_pending": 0.00000000, + "immature": 0.00000000 + }, + "lastprocessedblock": { + "hash": "0000003065c0669fff27edb4a71928cb48e5a6cfcdf06f491a83fd86822d18a6", + "height": 159592 + } +} +``` + + +You can also show transactions related to the wallet using `listtransactions` + +```sh +[online]$ ./src/bitcoin-cli -signet -rpcwallet="watch_only_wallet" listtransactions + +{ + ... +} +```
\ No newline at end of file diff --git a/src/Makefile.am b/src/Makefile.am index 8508d13b34..99b2184cf2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -404,6 +404,7 @@ libbitcoin_node_a_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ mapport.cpp \ @@ -944,6 +945,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ key.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b610dabd07..58c1da697d 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -92,6 +92,7 @@ BITCOIN_TESTS =\ test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/disconnected_transactions.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -111,6 +112,7 @@ BITCOIN_TESTS =\ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ + test/net_peer_connection_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index b001365ab3..5a11526471 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -610,8 +610,9 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; - LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); + const auto mapped_as{m_netgroupman.GetMappedAS(addr)}; + LogPrint(BCLog::ADDRMAN, "Added %s%s to new[%i][%i]\n", + addr.ToStringAddrPort(), (mapped_as ? strprintf(" mapped to AS%i", mapped_as) : ""), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -669,8 +670,9 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSecond } else { // move nId to the tried tables MakeTried(info, nId); - LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); + const auto mapped_as{m_netgroupman.GetMappedAS(addr)}; + LogPrint(BCLog::ADDRMAN, "Moved %s%s to tried[%i][%i]\n", + addr.ToStringAddrPort(), (mapped_as ? strprintf(" mapped to AS%i", mapped_as) : ""), tried_bucket, tried_bucket_pos); return true; } } diff --git a/src/banman.cpp b/src/banman.cpp index a96b7e3c53..9f668d76a3 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -28,7 +28,7 @@ BanMan::~BanMan() void BanMan::LoadBanlist() { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); @@ -52,16 +52,17 @@ void BanMan::DumpBanlist() banmap_t banmap; { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); SweepBanned(); - if (!BannedSetIsDirty()) return; + if (!m_is_dirty) return; banmap = m_banned; - SetBannedSetDirty(false); + m_is_dirty = false; } const auto start{SteadyClock::now()}; if (!m_ban_db.Write(banmap)) { - SetBannedSetDirty(true); + LOCK(m_banned_mutex); + m_is_dirty = true; } LogPrint(BCLog::NET, "Flushed %d banned node addresses/subnets to disk %dms\n", banmap.size(), @@ -71,7 +72,7 @@ void BanMan::DumpBanlist() void BanMan::ClearBanned() { { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); m_banned.clear(); m_is_dirty = true; } @@ -81,14 +82,14 @@ void BanMan::ClearBanned() bool BanMan::IsDiscouraged(const CNetAddr& net_addr) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); return m_discouraged.contains(net_addr.GetAddrBytes()); } bool BanMan::IsBanned(const CNetAddr& net_addr) { auto current_time = GetTime(); - LOCK(m_cs_banned); + LOCK(m_banned_mutex); for (const auto& it : m_banned) { CSubNet sub_net = it.first; CBanEntry ban_entry = it.second; @@ -103,7 +104,7 @@ bool BanMan::IsBanned(const CNetAddr& net_addr) bool BanMan::IsBanned(const CSubNet& sub_net) { auto current_time = GetTime(); - LOCK(m_cs_banned); + LOCK(m_banned_mutex); banmap_t::iterator i = m_banned.find(sub_net); if (i != m_banned.end()) { CBanEntry ban_entry = (*i).second; @@ -122,7 +123,7 @@ void BanMan::Ban(const CNetAddr& net_addr, int64_t ban_time_offset, bool since_u void BanMan::Discourage(const CNetAddr& net_addr) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); m_discouraged.insert(net_addr.GetAddrBytes()); } @@ -139,7 +140,7 @@ void BanMan::Ban(const CSubNet& sub_net, int64_t ban_time_offset, bool since_uni ban_entry.nBanUntil = (normalized_since_unix_epoch ? 0 : GetTime()) + normalized_ban_time_offset; { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_banned[sub_net].nBanUntil < ban_entry.nBanUntil) { m_banned[sub_net] = ban_entry; m_is_dirty = true; @@ -161,7 +162,7 @@ bool BanMan::Unban(const CNetAddr& net_addr) bool BanMan::Unban(const CSubNet& sub_net) { { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_banned.erase(sub_net) == 0) return false; m_is_dirty = true; } @@ -172,7 +173,7 @@ bool BanMan::Unban(const CSubNet& sub_net) void BanMan::GetBanned(banmap_t& banmap) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); // Sweep the banlist so expired bans are not returned SweepBanned(); banmap = m_banned; //create a thread safe copy @@ -180,7 +181,7 @@ void BanMan::GetBanned(banmap_t& banmap) void BanMan::SweepBanned() { - AssertLockHeld(m_cs_banned); + AssertLockHeld(m_banned_mutex); int64_t now = GetTime(); bool notify_ui = false; @@ -203,15 +204,3 @@ void BanMan::SweepBanned() m_client_interface->BannedListChanged(); } } - -bool BanMan::BannedSetIsDirty() -{ - LOCK(m_cs_banned); - return m_is_dirty; -} - -void BanMan::SetBannedSetDirty(bool dirty) -{ - LOCK(m_cs_banned); //reuse m_banned lock for the m_is_dirty flag - m_is_dirty = dirty; -} diff --git a/src/banman.h b/src/banman.h index 5a5f5677b0..c6df7ec3c3 100644 --- a/src/banman.h +++ b/src/banman.h @@ -60,40 +60,37 @@ class BanMan public: ~BanMan(); BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time); - void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Discourage(const CNetAddr& net_addr); - void ClearBanned(); + void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void Discourage(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void ClearBanned() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether net_addr is banned - bool IsBanned(const CNetAddr& net_addr); + bool IsBanned(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether sub_net is exactly banned - bool IsBanned(const CSubNet& sub_net); + bool IsBanned(const CSubNet& sub_net) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether net_addr is discouraged. - bool IsDiscouraged(const CNetAddr& net_addr); + bool IsDiscouraged(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); - bool Unban(const CNetAddr& net_addr); - bool Unban(const CSubNet& sub_net); - void GetBanned(banmap_t& banmap); - void DumpBanlist(); + bool Unban(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + bool Unban(const CSubNet& sub_net) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void GetBanned(banmap_t& banmap) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void DumpBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); private: - void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_banned); - bool BannedSetIsDirty(); - //!set the "dirty" flag for the banlist - void SetBannedSetDirty(bool dirty = true); + void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //!clean unused entries (if bantime has expired) - void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_cs_banned); + void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_banned_mutex); - RecursiveMutex m_cs_banned; - banmap_t m_banned GUARDED_BY(m_cs_banned); - bool m_is_dirty GUARDED_BY(m_cs_banned){false}; + Mutex m_banned_mutex; + banmap_t m_banned GUARDED_BY(m_banned_mutex); + bool m_is_dirty GUARDED_BY(m_banned_mutex){false}; CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; - CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; + CRollingBloomFilter m_discouraged GUARDED_BY(m_banned_mutex) {50000, 0.000001}; }; #endif // BITCOIN_BANMAN_H diff --git a/src/bench/disconnected_transactions.cpp b/src/bench/disconnected_transactions.cpp index 264c0aa1e8..91ce904dd9 100644 --- a/src/bench/disconnected_transactions.cpp +++ b/src/bench/disconnected_transactions.cpp @@ -73,7 +73,7 @@ static ReorgTxns CreateBlocks(size_t num_not_shared) static void Reorg(const ReorgTxns& reorg) { - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; // Disconnect block const auto evicted = disconnectpool.AddTransactionsFromBlock(reorg.disconnected_txns); assert(evicted.empty()); diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 211b4740be..29306b2229 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -107,12 +107,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - for (size_t i = 0; i < pool->vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); + for (const auto& tx : pool->txns_randomized) { + uint64_t shortid = cmpctblock.GetShortID(tx->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = tx; have_txn[idit->second] = true; mempool_count++; } else { diff --git a/src/consensus/validation.h b/src/consensus/validation.h index d5bf08cd61..bd3a5913c3 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -54,6 +54,8 @@ enum class TxValidationResult { TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction + TX_RECONSIDERABLE, //!< fails some policy, but might be acceptable if submitted in a (different) package + TX_UNKNOWN, //!< transaction was not validated because package failed }; /** A "reason" why a block was invalid, suitable for determining whether the diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 661406e122..c72dbf10bc 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -8,6 +8,7 @@ #include <crypto/hmac_sha256.h> #include <httpserver.h> #include <logging.h> +#include <netaddress.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <util/strencodings.h> diff --git a/src/init.cpp b/src/init.cpp index 42331d37e8..62e7e32272 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -343,11 +343,11 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - node.kernel.reset(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman.reset(); node.scheduler.reset(); + node.kernel.reset(); try { if (!fs::remove(GetPidFile(*node.args))) { @@ -489,7 +489,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", strprintf("Accept connections from outside (default: %u if no -proxy, -connect or -maxconnections=0)", DEFAULT_LISTEN), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> automatic connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1751,11 +1751,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nMaxConnections = nMaxConnections; - connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); - connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); - connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; + connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); diff --git a/src/kernel/disconnected_transactions.cpp b/src/kernel/disconnected_transactions.cpp new file mode 100644 index 0000000000..f865fed688 --- /dev/null +++ b/src/kernel/disconnected_transactions.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <kernel/disconnected_transactions.h> + +#include <assert.h> +#include <core_memusage.h> +#include <memusage.h> +#include <primitives/transaction.h> +#include <util/hasher.h> + +#include <memory> +#include <utility> + +// It's almost certainly a logic bug if we don't clear out queuedTx before +// destruction, as we add to it while disconnecting blocks, and then we +// need to re-process remaining transactions to ensure mempool consistency. +// For now, assert() that we've emptied out this object on destruction. +// This assert() can always be removed if the reorg-processing code were +// to be refactored such that this assumption is no longer true (for +// instance if there was some other way we cleaned up the mempool after a +// reorg, besides draining this object). +DisconnectedBlockTransactions::~DisconnectedBlockTransactions() +{ + assert(queuedTx.empty()); + assert(iters_by_txid.empty()); + assert(cachedInnerUsage == 0); +} + +std::vector<CTransactionRef> DisconnectedBlockTransactions::LimitMemoryUsage() +{ + std::vector<CTransactionRef> evicted; + + while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) { + evicted.emplace_back(queuedTx.front()); + cachedInnerUsage -= RecursiveDynamicUsage(queuedTx.front()); + iters_by_txid.erase(queuedTx.front()->GetHash()); + queuedTx.pop_front(); + } + return evicted; +} + +size_t DisconnectedBlockTransactions::DynamicMemoryUsage() const +{ + return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx); +} + +[[nodiscard]] std::vector<CTransactionRef> DisconnectedBlockTransactions::AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx) +{ + iters_by_txid.reserve(iters_by_txid.size() + vtx.size()); + for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) { + auto it = queuedTx.insert(queuedTx.end(), *block_it); + auto [_, inserted] = iters_by_txid.emplace((*block_it)->GetHash(), it); + assert(inserted); // callers may never pass multiple transactions with the same txid + cachedInnerUsage += RecursiveDynamicUsage(*block_it); + } + return LimitMemoryUsage(); +} + +void DisconnectedBlockTransactions::removeForBlock(const std::vector<CTransactionRef>& vtx) +{ + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (const auto& tx : vtx) { + auto iter = iters_by_txid.find(tx->GetHash()); + if (iter != iters_by_txid.end()) { + auto list_iter = iter->second; + iters_by_txid.erase(iter); + cachedInnerUsage -= RecursiveDynamicUsage(*list_iter); + queuedTx.erase(list_iter); + } + } +} + +void DisconnectedBlockTransactions::clear() +{ + cachedInnerUsage = 0; + iters_by_txid.clear(); + queuedTx.clear(); +} + +std::list<CTransactionRef> DisconnectedBlockTransactions::take() +{ + std::list<CTransactionRef> ret = std::move(queuedTx); + clear(); + return ret; +} diff --git a/src/kernel/disconnected_transactions.h b/src/kernel/disconnected_transactions.h index 7db39ba5ca..401ec435e6 100644 --- a/src/kernel/disconnected_transactions.h +++ b/src/kernel/disconnected_transactions.h @@ -5,8 +5,6 @@ #ifndef BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H #define BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H -#include <core_memusage.h> -#include <memusage.h> #include <primitives/transaction.h> #include <util/hasher.h> @@ -14,8 +12,8 @@ #include <unordered_map> #include <vector> -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000; +/** Maximum bytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_BYTES{20'000'000}; /** * DisconnectedBlockTransactions @@ -38,8 +36,7 @@ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000; */ class DisconnectedBlockTransactions { private: - /** Cached dynamic memory usage for the CTransactions (memory for the shared pointers is - * included in the container calculations). */ + /** Cached dynamic memory usage for the `CTransactionRef`s */ uint64_t cachedInnerUsage = 0; const size_t m_max_mem_usage; std::list<CTransactionRef> queuedTx; @@ -47,39 +44,15 @@ private: std::unordered_map<uint256, TxList::iterator, SaltedTxidHasher> iters_by_txid; /** Trim the earliest-added entries until we are within memory bounds. */ - std::vector<CTransactionRef> LimitMemoryUsage() - { - std::vector<CTransactionRef> evicted; - - while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) { - evicted.emplace_back(queuedTx.front()); - cachedInnerUsage -= RecursiveDynamicUsage(*queuedTx.front()); - iters_by_txid.erase(queuedTx.front()->GetHash()); - queuedTx.pop_front(); - } - return evicted; - } + std::vector<CTransactionRef> LimitMemoryUsage(); public: - DisconnectedBlockTransactions(size_t max_mem_usage) : m_max_mem_usage{max_mem_usage} {} + DisconnectedBlockTransactions(size_t max_mem_usage) + : m_max_mem_usage{max_mem_usage} {} - // It's almost certainly a logic bug if we don't clear out queuedTx before - // destruction, as we add to it while disconnecting blocks, and then we - // need to re-process remaining transactions to ensure mempool consistency. - // For now, assert() that we've emptied out this object on destruction. - // This assert() can always be removed if the reorg-processing code were - // to be refactored such that this assumption is no longer true (for - // instance if there was some other way we cleaned up the mempool after a - // reorg, besides draining this object). - ~DisconnectedBlockTransactions() { - assert(queuedTx.empty()); - assert(iters_by_txid.empty()); - assert(cachedInnerUsage == 0); - } + ~DisconnectedBlockTransactions(); - size_t DynamicMemoryUsage() const { - return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx); - } + size_t DynamicMemoryUsage() const; /** Add transactions from the block, iterating through vtx in reverse order. Callers should call * this function for blocks in descending order by block height. @@ -88,50 +61,16 @@ public: * corresponding entry in iters_by_txid. * @returns vector of transactions that were evicted for size-limiting. */ - [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx) - { - iters_by_txid.reserve(iters_by_txid.size() + vtx.size()); - for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) { - auto it = queuedTx.insert(queuedTx.end(), *block_it); - iters_by_txid.emplace((*block_it)->GetHash(), it); - cachedInnerUsage += RecursiveDynamicUsage(**block_it); - } - return LimitMemoryUsage(); - } + [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx); /** Remove any entries that are in this block. */ - void removeForBlock(const std::vector<CTransactionRef>& vtx) - { - // Short-circuit in the common case of a block being added to the tip - if (queuedTx.empty()) { - return; - } - for (const auto& tx : vtx) { - auto iter = iters_by_txid.find(tx->GetHash()); - if (iter != iters_by_txid.end()) { - auto list_iter = iter->second; - iters_by_txid.erase(iter); - cachedInnerUsage -= RecursiveDynamicUsage(**list_iter); - queuedTx.erase(list_iter); - } - } - } + void removeForBlock(const std::vector<CTransactionRef>& vtx); size_t size() const { return queuedTx.size(); } - void clear() - { - cachedInnerUsage = 0; - iters_by_txid.clear(); - queuedTx.clear(); - } + void clear(); /** Clear all data structures and return the list of transactions. */ - std::list<CTransactionRef> take() - { - std::list<CTransactionRef> ret = std::move(queuedTx); - clear(); - return ret; - } + std::list<CTransactionRef> take(); }; #endif // BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 1f175a5ccf..7c905ca4f4 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -83,7 +83,7 @@ private: const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block - LockPoints lockPoints; //!< Track the height and time at which tx was final + mutable LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these @@ -151,7 +151,7 @@ public: } // Update the LockPoints after a reorg - void UpdateLockPoints(const LockPoints& lp) + void UpdateLockPoints(const LockPoints& lp) const { lockPoints = lp; } @@ -172,8 +172,10 @@ public: Parents& GetMemPoolParents() const { return m_parents; } Children& GetMemPoolChildren() const { return m_children; } - mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable size_t idx_randomized; //!< Index in mempool's txns_randomized mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; +using CTxMemPoolEntryRef = CTxMemPoolEntry::CTxMemPoolEntryRef; + #endif // BITCOIN_KERNEL_MEMPOOL_ENTRY_H diff --git a/src/net.cpp b/src/net.cpp index 16075efdbb..4aee78846a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -417,21 +417,25 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? GetDefaultPort(pszDest) : m_params.GetDefaultPort()}; if (pszDest) { - const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; if (!resolved.empty()) { - const CService& rnd{resolved[GetRand(resolved.size())]}; - addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; - if (!addrConnect.IsValid()) { - LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); - return nullptr; - } - // It is possible that we already have a connection to the IP/port pszDest resolved to. - // In that case, drop the connection that was just created. - LOCK(m_nodes_mutex); - CNode* pnode = FindNode(static_cast<CService>(addrConnect)); - if (pnode) { - LogPrintf("Failed to open new connection, already connected\n"); - return nullptr; + Shuffle(resolved.begin(), resolved.end(), FastRandomContext()); + // If the connection is made by name, it can be the case that the name resolves to more than one address. + // We don't want to connect any more of them if we are already connected to one + for (const auto& r : resolved) { + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(r), NODE_NONE}; + if (!addrConnect.IsValid()) { + LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); + return nullptr; + } + // It is possible that we already have a connection to the IP/port pszDest resolved to. + // In that case, drop the connection that was just created. + LOCK(m_nodes_mutex); + CNode* pnode = FindNode(static_cast<CService>(addrConnect)); + if (pnode) { + LogPrintf("Not opening a connection to %s, already connected to %s\n", pszDest, addrConnect.ToStringAddrPort()); + return nullptr; + } } } } @@ -1730,7 +1734,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const CAddress& addr) { int nInbound = 0; - int nMaxInbound = nMaxConnections - m_max_outbound; AddWhitelistPermissionFlags(permission_flags, addr); if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::Implicit)) { @@ -1776,13 +1779,13 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. bool discouraged = m_banman && m_banman->IsDiscouraged(addr); - if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= nMaxInbound && discouraged) + if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= m_max_inbound && discouraged) { LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToStringAddrPort()); return; } - if (nInbound >= nMaxInbound) + if (nInbound >= m_max_inbound) { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection @@ -2733,7 +2736,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. - const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(m_max_automatic_connections - 1, 2)}; // Use BIP324 transport when both us and them have NODE_V2_P2P set. const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*strDest=*/nullptr, conn_type, use_v2transport); @@ -2754,7 +2757,7 @@ std::vector<CAddress> CConnman::GetCurrentBlockRelayOnlyConns() const return ret; } -std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const +std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo(bool include_connected) const { std::vector<AddedNodeInfo> ret; @@ -2789,6 +2792,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is an IP:port auto it = mapConnected.find(service); if (it != mapConnected.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = service; addedNode.fConnected = true; addedNode.fInbound = it->second; @@ -2797,6 +2803,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is a name auto it = mapConnectedByName.find(addr.m_added_node); if (it != mapConnectedByName.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = it->second.second; addedNode.fConnected = true; addedNode.fInbound = it->second.first; @@ -2815,21 +2824,19 @@ void CConnman::ThreadOpenAddedConnections() while (true) { CSemaphoreGrant grant(*semAddnode); - std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(/*include_connected=*/false); bool tried = false; for (const AddedNodeInfo& info : vInfo) { - if (!info.fConnected) { - if (!grant) { - // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting - // the addednodeinfo state might change. - break; - } - tried = true; - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; - grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); + if (!grant) { + // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting + // the addednodeinfo state might change. + break; } + tried = true; + CAddress addr(CService(), NODE_NONE); + OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); } // Retry every 60 seconds if a connection was attempted, otherwise two seconds if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) @@ -3208,18 +3215,17 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = std::make_unique<CSemaphore>(std::min(m_max_outbound, nMaxConnections)); + semOutbound = std::make_unique<CSemaphore>(std::min(m_max_automatic_outbound, m_max_automatic_connections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = std::make_unique<CSemaphore>(nMaxAddnode); + semAddnode = std::make_unique<CSemaphore>(m_max_addnode); } // // Start threads // assert(m_msgproc); - InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -3291,16 +3297,16 @@ void CConnman::Interrupt() condMsgProc.notify_all(); interruptNet(); - InterruptSocks5(true); + g_socks5_interrupt(); if (semOutbound) { - for (int i=0; i<m_max_outbound; i++) { + for (int i=0; i<m_max_automatic_outbound; i++) { semOutbound->post(); } } if (semAddnode) { - for (int i=0; i<nMaxAddnode; i++) { + for (int i=0; i<m_max_addnode; i++) { semAddnode->post(); } } @@ -3426,9 +3432,12 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const AddedNodeParams& add) { + const CService resolved(LookupNumeric(add.m_added_node, GetDefaultPort(add.m_added_node))); + const bool resolved_is_valid{resolved.IsValid()}; + LOCK(m_added_nodes_mutex); for (const auto& it : m_added_node_params) { - if (add.m_added_node == it.m_added_node) return false; + if (add.m_added_node == it.m_added_node || (resolved_is_valid && resolved == LookupNumeric(it.m_added_node, GetDefaultPort(it.m_added_node)))) return false; } m_added_node_params.push_back(add); @@ -1045,11 +1045,7 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - int nMaxConnections = 0; - int m_max_outbound_full_relay = 0; - int m_max_outbound_block_relay = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; + int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; BanMan* m_banman = nullptr; @@ -1076,13 +1072,12 @@ public: AssertLockNotHeld(m_total_bytes_sent_mutex); nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); - m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; + m_max_automatic_connections = connOptions.m_max_automatic_connections; + m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); + m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); + m_max_automatic_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + m_max_feeler; + m_max_inbound = std::max(0, m_max_automatic_connections - m_max_automatic_outbound); m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; - m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; m_client_interface = connOptions.uiInterface; m_banman = connOptions.m_banman; m_msgproc = connOptions.m_msgproc; @@ -1189,7 +1184,7 @@ public: bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); - std::vector<AddedNodeInfo> GetAddedNodeInfo() const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + std::vector<AddedNodeInfo> GetAddedNodeInfo(bool include_connected) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); /** * Attempts to open a connection. Currently only used from tests. @@ -1466,7 +1461,18 @@ private: std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; - int nMaxConnections; + + /** + * Maximum number of automatic connections permitted, excluding manual + * connections but including inbounds. May be changed by the user and is + * potentially limited by the operating system (number of file descriptors). + */ + int m_max_automatic_connections; + + /* + * Maximum number of peers by connection type. Might vary from defaults + * based on -maxconnections init value. + */ // How many full-relay (tx, block, addr) outbound peers we want int m_max_outbound_full_relay; @@ -1475,9 +1481,11 @@ private: // We do not relay tx or addr messages with these peers int m_max_outbound_block_relay; - int nMaxAddnode; - int nMaxFeeler; - int m_max_outbound; + int m_max_addnode{MAX_ADDNODE_CONNECTIONS}; + int m_max_feeler{MAX_FEELER_CONNECTIONS}; + int m_max_automatic_outbound; + int m_max_inbound; + bool m_use_addrman_outgoing; CClientUIInterface* m_client_interface; NetEventsInterface* m_msgproc; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7fcc399151..c9596a36b8 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1820,6 +1820,8 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: case TxValidationResult::TX_NO_MEMPOOL: + case TxValidationResult::TX_RECONSIDERABLE: + case TxValidationResult::TX_UNKNOWN: break; } return false; @@ -3770,7 +3772,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<CAddress> vAddr; - vRecv >> WithParams(ser_params, vAddr); + vRecv >> ser_params(vAddr); if (!SetupAddressRelay(pfrom, *peer)) { LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); @@ -5375,16 +5377,12 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros // No addr messages to send if (peer.m_addrs_to_send.empty()) return; - const char* msg_type; - CNetAddr::Encoding ser_enc; + CNetMsgMaker mm(node.GetCommonVersion()); if (peer.m_wants_addrv2) { - msg_type = NetMsgType::ADDRV2; - ser_enc = CNetAddr::Encoding::V2; + m_connman.PushMessage(&node, mm.Make(NetMsgType::ADDRV2, CAddress::V2_NETWORK(peer.m_addrs_to_send))); } else { - msg_type = NetMsgType::ADDR; - ser_enc = CNetAddr::Encoding::V1; + m_connman.PushMessage(&node, mm.Make(NetMsgType::ADDR, CAddress::V1_NETWORK(peer.m_addrs_to_send))); } - m_connman.PushMessage(&node, CNetMsgMaker(node.GetCommonVersion()).Make(msg_type, WithParams(CAddress::SerParams{{ser_enc}, CAddress::Format::Network}, peer.m_addrs_to_send))); peer.m_addrs_to_send.clear(); // we only send the big addr message once diff --git a/src/netbase.cpp b/src/netbase.cpp index 5a609a872e..9fbd9f7dea 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -30,7 +30,7 @@ bool fNameLookup = DEFAULT_NAME_LOOKUP; // Need ample time for negotiation for very slow proxies such as Tor std::chrono::milliseconds g_socks5_recv_timeout = 20s; -static std::atomic<bool> interruptSocks5Recv(false); +CThreadInterrupt g_socks5_interrupt; ReachableNets g_reachable_nets; @@ -271,7 +271,7 @@ enum class IntrRecvError { * IntrRecvError::OK only if all of the specified number of bytes were * read. * - * @see This function can be interrupted by calling InterruptSocks5(bool). + * @see This function can be interrupted by calling g_socks5_interrupt(). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) @@ -299,8 +299,9 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::m return IntrRecvError::NetworkError; } } - if (interruptSocks5Recv) + if (g_socks5_interrupt) { return IntrRecvError::Interrupted; + } curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; @@ -333,103 +334,93 @@ static std::string Socks5ErrorString(uint8_t err) bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* auth, const Sock& sock) { - IntrRecvError recvr; - LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); - if (strDest.size() > 255) { - return error("Hostname too long"); - } - // Construct the version identifier/method selection message - std::vector<uint8_t> vSocks5Init; - vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol - if (auth) { - vSocks5Init.push_back(0x02); // 2 method identifiers follow... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - vSocks5Init.push_back(SOCKS5Method::USER_PASS); - } else { - vSocks5Init.push_back(0x01); // 1 method identifier follows... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - } - ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5Init.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet1[2]; - if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); - return false; - } - if (pchRet1[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to initialize"); - } - if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { - // Perform username/password authentication (as described in RFC1929) - std::vector<uint8_t> vAuth; - vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation - if (auth->username.size() > 255 || auth->password.size() > 255) - return error("Proxy username or password too long"); - vAuth.push_back(auth->username.size()); - vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); - vAuth.push_back(auth->password.size()); - vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vAuth.size()) { - return error("Error sending authentication to proxy"); - } - LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); - uint8_t pchRetA[2]; - if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading proxy authentication response"); + try { + IntrRecvError recvr; + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); + if (strDest.size() > 255) { + return error("Hostname too long"); } - if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - return error("Proxy authentication unsuccessful"); + // Construct the version identifier/method selection message + std::vector<uint8_t> vSocks5Init; + vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol + if (auth) { + vSocks5Init.push_back(0x02); // 2 method identifiers follow... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); + vSocks5Init.push_back(SOCKS5Method::USER_PASS); + } else { + vSocks5Init.push_back(0x01); // 1 method identifier follows... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { - // Perform no authentication - } else { - return error("Proxy requested wrong authentication method %02x", pchRet1[1]); - } - std::vector<uint8_t> vSocks5; - vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version - vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT - vSocks5.push_back(0x00); // RSV Reserved must be 0 - vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME - vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function - vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); - vSocks5.push_back((port >> 8) & 0xFF); - vSocks5.push_back((port >> 0) & 0xFF); - ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { - if (recvr == IntrRecvError::Timeout) { - /* If a timeout happens here, this effectively means we timed out while connecting - * to the remote node. This is very common for Tor, so do not print an - * error message. */ + sock.SendComplete(vSocks5Init, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet1[2]; + if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; + } + if (pchRet1[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to initialize"); + } + if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { + // Perform username/password authentication (as described in RFC1929) + std::vector<uint8_t> vAuth; + vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation + if (auth->username.size() > 255 || auth->password.size() > 255) + return error("Proxy username or password too long"); + vAuth.push_back(auth->username.size()); + vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); + vAuth.push_back(auth->password.size()); + vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); + sock.SendComplete(vAuth, g_socks5_recv_timeout, g_socks5_interrupt); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + uint8_t pchRetA[2]; + if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading proxy authentication response"); + } + if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { + return error("Proxy authentication unsuccessful"); + } + } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { + // Perform no authentication } else { - return error("Error while reading proxy response"); + return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } - } - if (pchRet2[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to accept request"); - } - if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { - // Failures to connect to a peer that are not proxy errors - LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); - return false; - } - if (pchRet2[2] != 0x00) { // Reserved field must be 0 - return error("Error: malformed proxy response"); - } - uint8_t pchRet3[256]; - switch (pchRet2[3]) - { + std::vector<uint8_t> vSocks5; + vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version + vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT + vSocks5.push_back(0x00); // RSV Reserved must be 0 + vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME + vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function + vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); + vSocks5.push_back((port >> 8) & 0xFF); + vSocks5.push_back((port >> 0) & 0xFF); + sock.SendComplete(vSocks5, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet2[4]; + if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { + if (recvr == IntrRecvError::Timeout) { + /* If a timeout happens here, this effectively means we timed out while connecting + * to the remote node. This is very common for Tor, so do not print an + * error message. */ + return false; + } else { + return error("Error while reading proxy response"); + } + } + if (pchRet2[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to accept request"); + } + if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { + // Failures to connect to a peer that are not proxy errors + LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); + return false; + } + if (pchRet2[2] != 0x00) { // Reserved field must be 0 + return error("Error: malformed proxy response"); + } + uint8_t pchRet3[256]; + switch (pchRet2[3]) { case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, g_socks5_recv_timeout, sock); break; case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, g_socks5_recv_timeout, sock); break; - case SOCKS5Atyp::DOMAINNAME: - { + case SOCKS5Atyp::DOMAINNAME: { recvr = InterruptibleRecv(pchRet3, 1, g_socks5_recv_timeout, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); @@ -439,15 +430,18 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a break; } default: return error("Error: malformed proxy response"); + } + if (recvr != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); + return true; + } catch (const std::runtime_error& e) { + return error("Error during SOCKS5 proxy handshake: %s", e.what()); } - if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); - return true; } std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) @@ -681,11 +675,6 @@ CSubNet LookupSubNet(const std::string& subnet_str) return subnet; } -void InterruptSocks5(bool interrupt) -{ - interruptSocks5Recv = interrupt; -} - bool IsBadPort(uint16_t port) { /* Don't forget to update doc/p2p-bad-ports.md if you change this list. */ diff --git a/src/netbase.h b/src/netbase.h index d51f63fd81..8523f59b4d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -13,6 +13,7 @@ #include <netaddress.h> #include <serialize.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <functional> #include <memory> @@ -274,7 +275,10 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -void InterruptSocks5(bool interrupt); +/** + * Interrupt SOCKS5 reads or writes. + */ +extern CThreadInterrupt g_socks5_interrupt; /** * Connect to a specified destination service through an already connected diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f6dbe4f008..4c83df7eca 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -648,8 +648,9 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - auto it = m_node.mempool->GetIter(txid); - return it && (*it)->GetCountWithDescendants() > 1; + const auto entry{m_node.mempool->GetEntry(Txid::FromUint256(txid))}; + if (entry == nullptr) return false; + return entry->GetCountWithDescendants() > 1; } bool broadcastTransaction(const CTransactionRef& tx, const CAmount& max_tx_fee, @@ -702,9 +703,9 @@ public: if (!m_node.mempool) return true; LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); - const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); - return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); + std::string err_string; + return m_node.mempool->CheckPackageLimits({tx}, entry.GetTxSize(), err_string); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { @@ -806,7 +807,7 @@ public: { if (!m_node.mempool) return; LOCK2(::cs_main, m_node.mempool->cs); - for (const CTxMemPoolEntry& entry : m_node.mempool->mapTx) { + for (const CTxMemPoolEntry& entry : m_node.mempool->entryAll()) { notifications.transactionAddedToMempool(entry.GetSharedTx()); } } diff --git a/src/node/miner.cpp b/src/node/miner.cpp index caa2991819..ce5452d1f9 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -188,7 +188,7 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) { for (CTxMemPool::setEntries::iterator iit = testSet.begin(); iit != testSet.end(); ) { // Only test txs not already in the block - if (inBlock.count(*iit)) { + if (inBlock.count((*iit)->GetSharedTx()->GetHash())) { testSet.erase(iit++); } else { iit++; @@ -229,7 +229,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) ++nBlockTx; nBlockSigOpsCost += iter->GetSigOpCost(); nFees += iter->GetFee(); - inBlock.insert(iter); + inBlock.insert(iter->GetSharedTx()->GetHash()); bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { @@ -298,7 +298,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // because some of their txs are already in the block indexed_modified_transaction_set mapModifiedTx; // Keep track of entries that failed inclusion, to avoid duplicate work - CTxMemPool::setEntries failedTx; + std::set<Txid> failedTx; CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; @@ -326,7 +326,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (mi != mempool.mapTx.get<ancestor_score>().end()) { auto it = mempool.mapTx.project<0>(mi); assert(it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) { + if (mapModifiedTx.count(it) || inBlock.count(it->GetSharedTx()->GetHash()) || failedTx.count(it->GetSharedTx()->GetHash())) { ++mi; continue; } @@ -360,7 +360,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't // contain anything that is inBlock. - assert(!inBlock.count(iter)); + assert(!inBlock.count(iter->GetSharedTx()->GetHash())); uint64_t packageSize = iter->GetSizeWithAncestors(); CAmount packageFees = iter->GetModFeesWithAncestors(); @@ -382,7 +382,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // we must erase failed entries so that we can consider the // next best entry on the next loop iteration mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } ++nConsecutiveFailed; @@ -404,7 +404,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (!TestPackageTransactions(ancestors)) { if (fUsingModified) { mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } continue; } diff --git a/src/node/miner.h b/src/node/miner.h index 4173521585..06a917228d 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -142,7 +142,7 @@ private: uint64_t nBlockTx; uint64_t nBlockSigOpsCost; CAmount nFees; - CTxMemPool::setEntries inBlock; + std::unordered_set<Txid, SaltedTxidHasher> inBlock; // Chain context for the block int nHeight; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp index 2827242f96..58422c4439 100644 --- a/src/node/mini_miner.cpp +++ b/src/node/mini_miner.cpp @@ -4,9 +4,14 @@ #include <node/mini_miner.h> +#include <boost/multi_index/detail/hash_index_iterator.hpp> +#include <boost/operators.hpp> #include <consensus/amount.h> #include <policy/feerate.h> #include <primitives/transaction.h> +#include <sync.h> +#include <txmempool.h> +#include <uint256.h> #include <util/check.h> #include <algorithm> @@ -72,7 +77,12 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou // Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced. for (const auto& txiter : cluster) { if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { - auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter)); + auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), + MiniMinerMempoolEntry{/*tx_in=*/txiter->GetSharedTx(), + /*vsize_self=*/txiter->GetTxSize(), + /*vsize_ancestor=*/txiter->GetSizeWithAncestors(), + /*fee_self=*/txiter->GetModifiedFee(), + /*fee_ancestor=*/txiter->GetModFeesWithAncestors()}); m_entries.push_back(mapiter); } else { auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); @@ -122,6 +132,48 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou SanityCheck(); } +MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, + const std::map<Txid, std::set<Txid>>& descendant_caches) +{ + for (const auto& entry : manual_entries) { + const auto& txid = entry.GetTx().GetHash(); + // We need to know the descendant set of every transaction. + if (!Assume(descendant_caches.count(txid) > 0)) { + m_ready_to_calculate = false; + return; + } + // Just forward these args onto MiniMinerMempoolEntry + auto [mapiter, success] = m_entries_by_txid.emplace(txid, entry); + // Txids must be unique; this txid shouldn't already be an entry in m_entries_by_txid + if (Assume(success)) m_entries.push_back(mapiter); + } + // Descendant cache is already built, but we need to translate them to m_entries_by_txid iters. + for (const auto& [txid, desc_txids] : descendant_caches) { + // Descendant cache should include at least the tx itself. + if (!Assume(!desc_txids.empty())) { + m_ready_to_calculate = false; + return; + } + std::vector<MockEntryMap::iterator> descendants; + for (const auto& desc_txid : desc_txids) { + auto desc_it{m_entries_by_txid.find(desc_txid)}; + // Descendants should only include transactions with corresponding entries. + if (!Assume(desc_it != m_entries_by_txid.end())) { + m_ready_to_calculate = false; + return; + } else { + descendants.emplace_back(desc_it); + } + } + m_descendant_set_by_txid.emplace(txid, descendants); + } + Assume(m_to_be_replaced.empty()); + Assume(m_requested_outpoints_by_txid.empty()); + Assume(m_bump_fees.empty()); + Assume(m_inclusion_order.empty()); + SanityCheck(); +} + // Compare by min(ancestor feerate, individual feerate), then iterator // // Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate @@ -201,8 +253,10 @@ void MiniMiner::SanityCheck() const [&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();})); } -void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) +void MiniMiner::BuildMockTemplate(std::optional<CFeeRate> target_feerate) { + const auto num_txns{m_entries_by_txid.size()}; + uint32_t sequence_num{0}; while (!m_entries_by_txid.empty()) { // Sort again, since transaction removal may change some m_entries' ancestor feerates. std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator()); @@ -213,7 +267,8 @@ void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors(); const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors(); // Stop here. Everything that didn't "make it into the block" has bumpfee. - if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) { + if (target_feerate.has_value() && + ancestor_package_fee < target_feerate->GetFee(ancestor_package_size)) { break; } @@ -237,14 +292,32 @@ void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) to_process.erase(iter); } } + // Track the order in which transactions were selected. + for (const auto& ancestor : ancestors) { + m_inclusion_order.emplace(Txid::FromUint256(ancestor->first), sequence_num); + } DeleteAncestorPackage(ancestors); SanityCheck(); + ++sequence_num; + } + if (!target_feerate.has_value()) { + Assume(m_in_block.size() == num_txns); + } else { + Assume(m_in_block.empty() || m_total_fees >= target_feerate->GetFee(m_total_vsize)); } - Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize)); + Assume(m_in_block.empty() || sequence_num > 0); + Assume(m_in_block.size() == m_inclusion_order.size()); // Do not try to continue building the block template with a different feerate. m_ready_to_calculate = false; } + +std::map<Txid, uint32_t> MiniMiner::Linearize() +{ + BuildMockTemplate(std::nullopt); + return m_inclusion_order; +} + std::map<COutPoint, CAmount> MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate) { if (!m_ready_to_calculate) return {}; diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h index 9d9d66bf0b..de62c0af75 100644 --- a/src/node/mini_miner.h +++ b/src/node/mini_miner.h @@ -5,33 +5,45 @@ #ifndef BITCOIN_NODE_MINI_MINER_H #define BITCOIN_NODE_MINI_MINER_H -#include <txmempool.h> +#include <consensus/amount.h> +#include <primitives/transaction.h> +#include <uint256.h> +#include <map> #include <memory> #include <optional> +#include <set> #include <stdint.h> +#include <vector> + +class CFeeRate; +class CTxMemPool; namespace node { // Container for tracking updates to ancestor feerate as we include ancestors in the "block" class MiniMinerMempoolEntry { - const CAmount fee_individual; const CTransactionRef tx; const int64_t vsize_individual; - CAmount fee_with_ancestors; int64_t vsize_with_ancestors; + const CAmount fee_individual; + CAmount fee_with_ancestors; // This class must be constructed while holding mempool.cs. After construction, the object's // methods can be called without holding that lock. public: - explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) : - fee_individual{entry->GetModifiedFee()}, - tx{entry->GetSharedTx()}, - vsize_individual(entry->GetTxSize()), - fee_with_ancestors{entry->GetModFeesWithAncestors()}, - vsize_with_ancestors(entry->GetSizeWithAncestors()) + explicit MiniMinerMempoolEntry(const CTransactionRef& tx_in, + int64_t vsize_self, + int64_t vsize_ancestor, + CAmount fee_self, + CAmount fee_ancestor): + tx{tx_in}, + vsize_individual{vsize_self}, + vsize_with_ancestors{vsize_ancestor}, + fee_individual{fee_self}, + fee_with_ancestors{fee_ancestor} { } CAmount GetModifiedFee() const { return fee_individual; } @@ -55,8 +67,14 @@ struct IteratorComparator } }; -/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of - * mempool transactions, ignoring consensus rules, to calculate mining scores. */ +/** A minimal version of BlockAssembler, using the same ancestor set scoring algorithm. Allows us to + * run this algorithm on a limited set of transactions (e.g. subset of mempool or transactions that + * are not yet in mempool) instead of the entire mempool, ignoring consensus rules. + * Callers may use this to: + * - Calculate the "bump fee" needed to spend an unconfirmed UTXO at a given feerate + * - "Linearize" a list of transactions to see the order in which they would be selected for + * inclusion in a block + */ class MiniMiner { // When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve @@ -72,7 +90,11 @@ class MiniMiner // the same tx will have the same bumpfee. Excludes non-mempool transactions. std::map<uint256, std::vector<COutPoint>> m_requested_outpoints_by_txid; - // What we're trying to calculate. + // Txid to a number representing the order in which this transaction was included (smaller + // number = included earlier). Transactions included in an ancestor set together have the same + // sequence number. + std::map<Txid, uint32_t> m_inclusion_order; + // What we're trying to calculate. Outpoint to the fee needed to bring the transaction to the target feerate. std::map<COutPoint, CAmount> m_bump_fees; // The constructed block template @@ -102,14 +124,32 @@ public: /** Returns true if CalculateBumpFees may be called, false if not. */ bool IsReadyToCalculate() const { return m_ready_to_calculate; } - /** Build a block template until the target feerate is hit. */ - void BuildMockTemplate(const CFeeRate& target_feerate); + /** Build a block template until the target feerate is hit. If target_feerate is not given, + * builds a block template until all transactions have been selected. */ + void BuildMockTemplate(std::optional<CFeeRate> target_feerate); /** Returns set of txids in the block template if one has been constructed. */ std::set<uint256> GetMockTemplateTxids() const { return m_in_block; } + /** Constructor that takes a list of outpoints that may or may not belong to transactions in the + * mempool. Copies out information about the relevant transactions in the mempool into + * MiniMinerMempoolEntrys. + */ MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); + /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually. + * It is assumed that all entries are unique and their values are correct, otherwise results + * computed by MiniMiner may be incorrect. Callers should check IsReadyToCalculate() after + * construction. + * @param[in] descendant_caches A map from each transaction to the set of txids of this + * transaction's descendant set, including itself. Each tx in + * manual_entries must have a corresponding entry in this map, and + * all of the txids in a descendant set must correspond to a tx in + * manual_entries. + */ + MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, + const std::map<Txid, std::set<Txid>>& descendant_caches); + /** Construct a new block template and, for each outpoint corresponding to a transaction that * did not make it into the block, calculate the cost of bumping those transactions (and their * ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map @@ -120,6 +160,12 @@ public: * not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt * if it cannot be calculated. */ std::optional<CAmount> CalculateTotalBumpFees(const CFeeRate& target_feerate); + + /** Construct a new block template with all of the transactions and calculate the order in which + * they are selected. Returns the sequence number (lower = selected earlier) with which each + * transaction was selected, indexed by txid, or an empty map if it cannot be calculated. + */ + std::map<Txid, uint32_t> Linearize(); }; } // namespace node diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 87bfa4cfc3..9557594622 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -260,6 +260,11 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, unsigned int curFarBucket = maxbucketindex; unsigned int bestFarBucket = maxbucketindex; + // We'll always group buckets into sets that meet sufficientTxVal -- + // this ensures that we're using consistent groups between different + // confirmation targets. + double partialNum = 0; + bool foundAnswer = false; unsigned int bins = unconfTxs.size(); bool newBucketRange = true; @@ -275,6 +280,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, } curFarBucket = bucket; nConf += confAvg[periodTarget - 1][bucket]; + partialNum += txCtAvg[bucket]; totalNum += txCtAvg[bucket]; failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) @@ -284,7 +290,14 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // we can test for success // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) - if (totalNum >= sufficientTxVal / (1 - decay)) { + + if (partialNum < sufficientTxVal / (1 - decay)) { + // the buckets we've added in this round aren't sufficient + // so keep adding + continue; + } else { + partialNum = 0; // reset for the next range we'll add + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate diff --git a/src/policy/packages.cpp b/src/policy/packages.cpp index 47a9274a31..3a63a9fe46 100644 --- a/src/policy/packages.cpp +++ b/src/policy/packages.cpp @@ -6,16 +6,77 @@ #include <policy/policy.h> #include <primitives/transaction.h> #include <uint256.h> -#include <util/hasher.h> +#include <util/check.h> #include <algorithm> #include <cassert> #include <iterator> #include <memory> #include <numeric> -#include <unordered_set> -bool CheckPackage(const Package& txns, PackageValidationState& state) +/** IsTopoSortedPackage where a set of txids has been pre-populated. The set is assumed to be correct and + * is mutated within this function (even if return value is false). */ +bool IsTopoSortedPackage(const Package& txns, std::unordered_set<uint256, SaltedTxidHasher>& later_txids) +{ + // Avoid misusing this function: later_txids should contain the txids of txns. + Assume(txns.size() == later_txids.size()); + + // later_txids always contains the txids of this transaction and the ones that come later in + // txns. If any transaction's input spends a tx in that set, we've found a parent placed later + // than its child. + for (const auto& tx : txns) { + for (const auto& input : tx->vin) { + if (later_txids.find(input.prevout.hash) != later_txids.end()) { + // The parent is a subsequent transaction in the package. + return false; + } + } + // Avoid misusing this function: later_txids must contain every tx. + Assume(later_txids.erase(tx->GetHash()) == 1); + } + + // Avoid misusing this function: later_txids should have contained the txids of txns. + Assume(later_txids.empty()); + return true; +} + +bool IsTopoSortedPackage(const Package& txns) +{ + std::unordered_set<uint256, SaltedTxidHasher> later_txids; + std::transform(txns.cbegin(), txns.cend(), std::inserter(later_txids, later_txids.end()), + [](const auto& tx) { return tx->GetHash(); }); + + return IsTopoSortedPackage(txns, later_txids); +} + +bool IsConsistentPackage(const Package& txns) +{ + // Don't allow any conflicting transactions, i.e. spending the same inputs, in a package. + std::unordered_set<COutPoint, SaltedOutpointHasher> inputs_seen; + for (const auto& tx : txns) { + if (tx->vin.empty()) { + // This function checks consistency based on inputs, and we can't do that if there are + // no inputs. Duplicate empty transactions are also not consistent with one another. + // This doesn't create false negatives, as unconfirmed transactions are not allowed to + // have no inputs. + return false; + } + for (const auto& input : tx->vin) { + if (inputs_seen.find(input.prevout) != inputs_seen.end()) { + // This input is also present in another tx in the package. + return false; + } + } + // Batch-add all the inputs for a tx at a time. If we added them 1 at a time, we could + // catch duplicate inputs within a single tx. This is a more severe, consensus error, + // and we want to report that from CheckTransaction instead. + std::transform(tx->vin.cbegin(), tx->vin.cend(), std::inserter(inputs_seen, inputs_seen.end()), + [](const auto& input) { return input.prevout; }); + } + return true; +} + +bool IsWellFormedPackage(const Package& txns, PackageValidationState& state, bool require_sorted) { const unsigned int package_count = txns.size(); @@ -30,10 +91,6 @@ bool CheckPackage(const Package& txns, PackageValidationState& state) return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-too-large"); } - // Require the package to be sorted in order of dependency, i.e. parents appear before children. - // An unsorted package will fail anyway on missing-inputs, but it's better to quit earlier and - // fail on something less ambiguous (missing-inputs could also be an orphan or trying to - // spend nonexistent coins). std::unordered_set<uint256, SaltedTxidHasher> later_txids; std::transform(txns.cbegin(), txns.cend(), std::inserter(later_txids, later_txids.end()), [](const auto& tx) { return tx->GetHash(); }); @@ -44,30 +101,17 @@ bool CheckPackage(const Package& txns, PackageValidationState& state) return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-contains-duplicates"); } - for (const auto& tx : txns) { - for (const auto& input : tx->vin) { - if (later_txids.find(input.prevout.hash) != later_txids.end()) { - // The parent is a subsequent transaction in the package. - return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-sorted"); - } - } - later_txids.erase(tx->GetHash()); + // Require the package to be sorted in order of dependency, i.e. parents appear before children. + // An unsorted package will fail anyway on missing-inputs, but it's better to quit earlier and + // fail on something less ambiguous (missing-inputs could also be an orphan or trying to + // spend nonexistent coins). + if (require_sorted && !IsTopoSortedPackage(txns, later_txids)) { + return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-sorted"); } // Don't allow any conflicting transactions, i.e. spending the same inputs, in a package. - std::unordered_set<COutPoint, SaltedOutpointHasher> inputs_seen; - for (const auto& tx : txns) { - for (const auto& input : tx->vin) { - if (inputs_seen.find(input.prevout) != inputs_seen.end()) { - // This input is also present in another tx in the package. - return state.Invalid(PackageValidationResult::PCKG_POLICY, "conflict-in-package"); - } - } - // Batch-add all the inputs for a tx at a time. If we added them 1 at a time, we could - // catch duplicate inputs within a single tx. This is a more severe, consensus error, - // and we want to report that from CheckTransaction instead. - std::transform(tx->vin.cbegin(), tx->vin.cend(), std::inserter(inputs_seen, inputs_seen.end()), - [](const auto& input) { return input.prevout; }); + if (!IsConsistentPackage(txns)) { + return state.Invalid(PackageValidationResult::PCKG_POLICY, "conflict-in-package"); } return true; } diff --git a/src/policy/packages.h b/src/policy/packages.h index cf37857e4b..537d8476e2 100644 --- a/src/policy/packages.h +++ b/src/policy/packages.h @@ -9,8 +9,10 @@ #include <consensus/validation.h> #include <policy/policy.h> #include <primitives/transaction.h> +#include <util/hasher.h> #include <cstdint> +#include <unordered_set> #include <vector> /** Default maximum number of transactions in a package. */ @@ -49,13 +51,32 @@ using Package = std::vector<CTransactionRef>; class PackageValidationState : public ValidationState<PackageValidationResult> {}; +/** If any direct dependencies exist between transactions (i.e. a child spending the output of a + * parent), checks that all parents appear somewhere in the list before their respective children. + * No other ordering is enforced. This function cannot detect indirect dependencies (e.g. a + * transaction's grandparent if its parent is not present). + * @returns true if sorted. False if any tx spends the output of a tx that appears later in txns. + */ +bool IsTopoSortedPackage(const Package& txns); + +/** Checks that these transactions don't conflict, i.e., spend the same prevout. This includes + * checking that there are no duplicate transactions. Since these checks require looking at the inputs + * of a transaction, returns false immediately if any transactions have empty vin. + * + * Does not check consistency of a transaction with oneself; does not check if a transaction spends + * the same prevout multiple times (see bad-txns-inputs-duplicate in CheckTransaction()). + * + * @returns true if there are no conflicts. False if any two transactions spend the same prevout. + * */ +bool IsConsistentPackage(const Package& txns); + /** Context-free package policy checks: * 1. The number of transactions cannot exceed MAX_PACKAGE_COUNT. * 2. The total weight cannot exceed MAX_PACKAGE_WEIGHT. * 3. If any dependencies exist between transactions, parents must appear before children. * 4. Transactions cannot conflict, i.e., spend the same inputs. */ -bool CheckPackage(const Package& txns, PackageValidationState& state); +bool IsWellFormedPackage(const Package& txns, PackageValidationState& state, bool require_sorted); /** Context-free check that a package is exactly one child and its parents; not all parents need to * be present, but the package must not contain any transactions that are not the child's parents. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index d032b74008..76ab2b1a96 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -12,6 +12,7 @@ #include <tinyformat.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/rbf.h> @@ -35,7 +36,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. - const CTxMemPoolEntry entry{*pool.mapTx.find(tx.GetHash())}; + const auto& entry{*Assert(pool.GetEntry(tx.GetHash()))}; auto ancestors{pool.AssumeCalculateMemPoolAncestors(__func__, entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; diff --git a/src/protocol.h b/src/protocol.h index a58d671a70..e405253632 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -445,7 +445,7 @@ public: } // Invoke V1/V2 serializer for CService parent object. const auto ser_params{use_v2 ? CNetAddr::V2 : CNetAddr::V1}; - READWRITE(WithParams(ser_params, AsBase<CService>(obj))); + READWRITE(ser_params(AsBase<CService>(obj))); } //! Always included in serialization. The behavior is unspecified if the value is not representable as uint32_t. diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 67af62285d..7e24dbd3ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -531,6 +531,9 @@ void TransactionView::showDetails() TransactionDescDialog *dlg = new TransactionDescDialog(selection.at(0)); dlg->setAttribute(Qt::WA_DeleteOnClose); m_opened_dialogs.append(dlg); + connect(dlg, &QObject::destroyed, [this, dlg] { + m_opened_dialogs.removeOne(dlg); + }); dlg->show(); } } diff --git a/src/random.cpp b/src/random.cpp index 9bd8ff9f1a..ce4266a567 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -38,6 +38,9 @@ #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif +#if defined(HAVE_STRONG_GETAUXVAL) && defined(__aarch64__) +#include <sys/auxv.h> +#endif [[noreturn]] static void RandFailure() { @@ -173,6 +176,62 @@ static uint64_t GetRdSeed() noexcept #endif } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + +static bool g_rndr_supported = false; + +static void InitHardwareRand() +{ + if (getauxval(AT_HWCAP2) & HWCAP2_RNG) { + g_rndr_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rndr_supported) { + LogPrintf("Using RNDR and RNDRRS as additional entropy sources\n"); + } +} + +/** Read 64 bits of entropy using rndr. + * + * Must only be called when RNDR is supported. + */ +static uint64_t GetRNDR() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDR--Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_0; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + +/** Read 64 bits of entropy using rndrrs. + * + * Must only be called when RNDRRS is supported. + */ +static uint64_t GetRNDRRS() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDRRS--Reseeded-Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_1; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -191,6 +250,12 @@ static void SeedHardwareFast(CSHA512& hasher) noexcept { hasher.Write((const unsigned char*)&out, sizeof(out)); return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + uint64_t out = GetRNDR(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif } @@ -216,6 +281,14 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { } return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRNDRRS(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } #endif } diff --git a/src/rest.cpp b/src/rest.cpp index e094039366..71b7036980 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -26,6 +26,7 @@ #include <txmempool.h> #include <util/any.h> #include <util/check.h> +#include <util/strencodings.h> #include <validation.h> #include <version.h> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 23cef42d30..bf60eae433 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -21,6 +21,7 @@ #include <univalue.h> #include <util/fs.h> #include <util/moneystr.h> +#include <util/strencodings.h> #include <util/time.h> #include <utility> @@ -289,7 +290,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("descendantsize", e.GetSizeWithDescendants()); info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", e.GetTx().GetWitnessHash().ToString()); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -315,9 +316,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { + for (const CTxMemPoolEntry& child : e.GetMemPoolChildrenConst()) { spent.push_back(child.GetTx().GetHash().ToString()); } @@ -344,14 +343,13 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo } LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); + for (const CTxMemPoolEntry& e : pool.entryAll()) { UniValue info(UniValue::VOBJ); entryToJSON(pool, info, e); // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). // UniValue::pushKVEnd is used instead which currently is O(1). - o.pushKVEnd(hash.ToString(), info); + o.pushKVEnd(e.GetTx().GetHash().ToString(), info); } return o; } else { @@ -460,12 +458,12 @@ static RPCHelpMan getmempoolancestors() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *it, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; + auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; if (!fVerbose) { UniValue o(UniValue::VARR); @@ -521,15 +519,15 @@ static RPCHelpMan getmempooldescendants() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto it{mempool.GetIter(hash)}; + if (!it) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); + mempool.CalculateDescendants(*it, setDescendants); // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); + setDescendants.erase(*it); if (!fVerbose) { UniValue o(UniValue::VARR); @@ -573,14 +571,13 @@ static RPCHelpMan getmempoolentry() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); + entryToJSON(mempool, info, *entry); return info; }, }; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1a9052e008..c631132df2 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -16,6 +16,7 @@ #include <netbase.h> #include <node/context.h> #include <policy/settings.h> +#include <protocol.h> #include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/server_util.h> @@ -98,6 +99,18 @@ static RPCHelpMan ping() }; } +/** Returns, given services flags, a list of humanly readable (known) network services */ +static UniValue GetServicesNames(ServiceFlags services) +{ + UniValue servicesNames(UniValue::VARR); + + for (const auto& flag : serviceFlagsToStr(services)) { + servicesNames.push_back(flag); + } + + return servicesNames; +} + static RPCHelpMan getpeerinfo() { return RPCHelpMan{ @@ -487,7 +500,7 @@ static RPCHelpMan getaddednodeinfo() NodeContext& node = EnsureAnyNodeContext(request.context); const CConnman& connman = EnsureConnman(node); - std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(/*include_connected=*/true); if (!request.params[0].isNull()) { bool found = false; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index a11366bd47..63618ad202 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -20,6 +20,7 @@ #include <util/string.h> #include <util/translation.h> +#include <string_view> #include <tuple> const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; @@ -74,29 +75,29 @@ CAmount AmountFromValue(const UniValue& value, int decimals) return amount; } -uint256 ParseHashV(const UniValue& v, std::string strName) +uint256 ParseHashV(const UniValue& v, std::string_view name) { const std::string& strHex(v.get_str()); if (64 != strHex.length()) - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", strName, 64, strHex.length(), strHex)); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", name, 64, strHex.length(), strHex)); if (!IsHex(strHex)) // Note: IsHex("") is false - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be hexadecimal string (not '%s')", name, strHex)); return uint256S(strHex); } -uint256 ParseHashO(const UniValue& o, std::string strKey) +uint256 ParseHashO(const UniValue& o, std::string_view strKey) { return ParseHashV(o.find_value(strKey), strKey); } -std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name) { std::string strHex; if (v.isStr()) strHex = v.get_str(); if (!IsHex(strHex)) - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be hexadecimal string (not '%s')", name, strHex)); return ParseHex(strHex); } -std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey) { return ParseHexV(o.find_value(strKey), strKey); } @@ -1303,17 +1304,6 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl return ret; } -UniValue GetServicesNames(ServiceFlags services) -{ - UniValue servicesNames(UniValue::VARR); - - for (const auto& flag : serviceFlagsToStr(services)) { - servicesNames.push_back(flag); - } - - return servicesNames; -} - /** Convert a vector of bilingual strings to a UniValue::VARR containing their original untranslated values. */ [[nodiscard]] static UniValue BilingualStringsToUniValue(const std::vector<bilingual_str>& bilingual_strings) { diff --git a/src/rpc/util.h b/src/rpc/util.h index 392540ffad..d2137993de 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -9,7 +9,6 @@ #include <consensus/amount.h> #include <node/transaction.h> #include <outputtype.h> -#include <protocol.h> #include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -26,6 +25,7 @@ #include <map> #include <optional> #include <string> +#include <string_view> #include <type_traits> #include <utility> #include <variant> @@ -59,7 +59,6 @@ extern const std::string UNIX_EPOCH_TIME; extern const std::string EXAMPLE_ADDRESS[2]; class FillableSigningProvider; -class CPubKey; class CScript; struct Sections; @@ -91,10 +90,10 @@ void RPCTypeCheckObj(const UniValue& o, * Utilities: convert hex-encoded Values * (throws error if not hex). */ -uint256 ParseHashV(const UniValue& v, std::string strName); -uint256 ParseHashO(const UniValue& o, std::string strKey); -std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); -std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); +uint256 ParseHashV(const UniValue& v, std::string_view name); +uint256 ParseHashO(const UniValue& o, std::string_view strKey); +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name); +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey); /** * Validate and return a CAmount from a UniValue number or string. @@ -132,9 +131,6 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); -/** Returns, given services flags, a list of humanly readable (known) network services */ -UniValue GetServicesNames(ServiceFlags services); - /** * Serializing JSON objects depends on the outer type. Only arrays and * dictionaries can be nested in json. The top-level outer type is "NONE". diff --git a/src/serialize.h b/src/serialize.h index 8b15178ec0..b09d6fbe60 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -214,11 +214,11 @@ const Out& AsBase(const In& x) * } * }; * which would then be invoked as - * READWRITE(WithParams(BarParameter{...}, Using<FooFormatter>(obj.foo))) + * READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo))) * - * WithParams(parameter, obj) can be invoked anywhere in the call stack; it is + * parameter(obj) can be invoked anywhere in the call stack; it is * passed down recursively into all serialization code, until another - * WithParams overrides it. + * serialization parameter overrides it. * * Parameters will be implicitly converted where appropriate. This means that * "parent" serialization code can use a parameter that derives from, or is @@ -1183,17 +1183,6 @@ public: }; /** - * Return a wrapper around t that (de)serializes it with specified parameter params. - * - * See FORMATTER_METHODS_PARAMS for more information on serialization parameters. - */ -template <typename Params, typename T> -static auto WithParams(const Params& params, T&& t) -{ - return ParamsWrapper<Params, T>{params, t}; -} - -/** * Helper macro for SerParams structs * * Allows you define SerParams instances and then apply them directly @@ -1202,8 +1191,16 @@ static auto WithParams(const Params& params, T&& t) * constexpr SerParams FOO{....}; * ss << FOO(obj); */ -#define SER_PARAMS_OPFUNC \ - template <typename T> \ - auto operator()(T&& t) const { return WithParams(*this, t); } +#define SER_PARAMS_OPFUNC \ + /** \ + * Return a wrapper around t that (de)serializes it with specified parameter params. \ + * \ + * See FORMATTER_METHODS_PARAMS for more information on serialization parameters. \ + */ \ + template <typename T> \ + auto operator()(T&& t) const \ + { \ + return ParamsWrapper{*this, t}; \ + } #endif // BITCOIN_SERIALIZE_H diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 4348a20886..e4ef019daf 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -51,8 +51,8 @@ static CBlock BuildBlockTestCase() { } // Number of shared use_counts we expect for a tx we haven't touched -// (block + mempool + our copy from the GetSharedTx call) -constexpr long SHARED_TX_OFFSET{3}; +// (block + mempool entry + mempool txns_randomized + our copy from the GetSharedTx call) +constexpr long SHARED_TX_OFFSET{4}; BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2], MemPoolRemovalReason::REPLACED); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock CBlock block2; { @@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that partialBlock = tmp; } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); @@ -196,15 +196,15 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 txhash = block.vtx[2]->GetHash(); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -240,7 +240,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; @@ -253,9 +253,9 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) txhash = block.vtx[1]->GetHash(); block.vtx.clear(); block2.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6e740a4f53..0fef8c5906 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -147,9 +147,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = MAX_FEELER_CONNECTIONS; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; const auto time_init{GetTime<std::chrono::seconds>()}; SetMockTime(time_init); @@ -248,9 +246,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; - options.m_max_outbound_block_relay = max_outbound_block_relay; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; connman->Init(options); std::vector<CNode*> vNodes; diff --git a/src/test/disconnected_transactions.cpp b/src/test/disconnected_transactions.cpp new file mode 100644 index 0000000000..d4dc124b7b --- /dev/null +++ b/src/test/disconnected_transactions.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <boost/test/unit_test.hpp> +#include <core_memusage.h> +#include <kernel/disconnected_transactions.h> +#include <test/util/setup_common.h> + +BOOST_FIXTURE_TEST_SUITE(disconnected_transactions, TestChain100Setup) + +//! Tests that DisconnectedBlockTransactions limits its own memory properly +BOOST_AUTO_TEST_CASE(disconnectpool_memory_limits) +{ + // Use the coinbase transactions from TestChain100Setup. It doesn't matter whether these + // transactions would realistically be in a block together, they just need distinct txids and + // uniform size for this test to work. + std::vector<CTransactionRef> block_vtx(m_coinbase_txns); + BOOST_CHECK_EQUAL(block_vtx.size(), 100); + + // Roughly estimate sizes to sanity check that DisconnectedBlockTransactions::DynamicMemoryUsage + // is within an expected range. + + // Overhead for the hashmap depends on number of buckets + std::unordered_map<uint256, CTransaction*, SaltedTxidHasher> temp_map; + temp_map.reserve(1); + const size_t MAP_1{memusage::DynamicUsage(temp_map)}; + temp_map.reserve(100); + const size_t MAP_100{memusage::DynamicUsage(temp_map)}; + + const size_t TX_USAGE{RecursiveDynamicUsage(block_vtx.front())}; + for (const auto& tx : block_vtx) + BOOST_CHECK_EQUAL(RecursiveDynamicUsage(tx), TX_USAGE); + + // Our overall formula is unordered map overhead + usage per entry. + // Implementations may vary, but we're trying to guess the usage of data structures. + const size_t ENTRY_USAGE_ESTIMATE{ + TX_USAGE + // list entry: 2 pointers (next pointer and prev pointer) + element itself + + memusage::MallocUsage((2 * sizeof(void*)) + sizeof(decltype(block_vtx)::value_type)) + // unordered map: 1 pointer for the hashtable + key and value + + memusage::MallocUsage(sizeof(void*) + sizeof(decltype(temp_map)::key_type) + + sizeof(decltype(temp_map)::value_type))}; + + // DisconnectedBlockTransactions that's just big enough for 1 transaction. + { + DisconnectedBlockTransactions disconnectpool{MAP_1 + ENTRY_USAGE_ESTIMATE}; + // Add just 2 (and not all 100) transactions to keep the unordered map's hashtable overhead + // to a minimum and avoid all (instead of all but 1) transactions getting evicted. + std::vector<CTransactionRef> two_txns({block_vtx.at(0), block_vtx.at(1)}); + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(two_txns)}; + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAP_1 + ENTRY_USAGE_ESTIMATE); + + // Only 1 transaction can be kept + BOOST_CHECK_EQUAL(1, evicted_txns.size()); + // Transactions are added from back to front and eviction is FIFO. + BOOST_CHECK_EQUAL(block_vtx.at(1), evicted_txns.front()); + + disconnectpool.clear(); + } + + // DisconnectedBlockTransactions with a comfortable maximum memory usage so that nothing is evicted. + // Record usage so we can check size limiting in the next test. + size_t usage_full{0}; + { + const size_t USAGE_100_OVERESTIMATE{MAP_100 + ENTRY_USAGE_ESTIMATE * 100}; + DisconnectedBlockTransactions disconnectpool{USAGE_100_OVERESTIMATE}; + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)}; + BOOST_CHECK_EQUAL(evicted_txns.size(), 0); + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= USAGE_100_OVERESTIMATE); + + usage_full = disconnectpool.DynamicMemoryUsage(); + + disconnectpool.clear(); + } + + // DisconnectedBlockTransactions that's just a little too small for all of the transactions. + { + const size_t MAX_MEMUSAGE_99{usage_full - sizeof(void*)}; + DisconnectedBlockTransactions disconnectpool{MAX_MEMUSAGE_99}; + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)}; + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAX_MEMUSAGE_99); + + // Only 1 transaction needed to be evicted + BOOST_CHECK_EQUAL(1, evicted_txns.size()); + + // Transactions are added from back to front and eviction is FIFO. + // The last transaction of block_vtx should be the first to be evicted. + BOOST_CHECK_EQUAL(block_vtx.back(), evicted_txns.front()); + + disconnectpool.clear(); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 3e303ecc0f..612f113b14 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -10,21 +10,22 @@ #include <uint256.h> #include <cassert> -#include <cstdint> +#include <limits> #include <optional> -#include <string> #include <vector> FUZZ_TARGET(bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; CBloomFilter bloom_filter{ fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 10000000), 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.remaining_bytes() > 0, 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -37,6 +38,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { + good_data = false; return; } (void)bloom_filter.contains(*out_point); @@ -47,6 +49,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { + good_data = false; return; } (void)bloom_filter.contains(*u256); @@ -57,6 +60,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { + good_data = false; return; } const CTransaction tx{*mut_tx}; diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b088aa0bd7..1a8570ee8f 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,26 +2,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> -#include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> -#include <pubkey.h> +#include <script/interpreter.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <util/hasher.h> +#include <cassert> #include <cstdint> #include <limits> +#include <memory> #include <optional> +#include <stdexcept> #include <string> +#include <utility> #include <vector> namespace { @@ -44,12 +46,15 @@ void initialize_coins_view() FUZZ_TARGET(coins_view, .init = initialize_coins_view) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; + CCoinsView backend_coins_view; CCoinsViewCache coins_view_cache{&backend_coins_view, /*deterministic=*/true}; COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -95,6 +100,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!opt_out_point) { + good_data = false; return; } random_out_point = *opt_out_point; @@ -102,6 +108,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } random_coin = *opt_coin; @@ -109,6 +116,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!opt_mutable_transaction) { + good_data = false; return; } random_mutable_transaction = *opt_mutable_transaction; @@ -116,7 +124,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { CCoinsMapMemoryResource resource; CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { @@ -124,6 +133,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } else { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } coins_cache_entry.coin = *opt_coin; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0dab2a2e97..551e1c526d 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -121,7 +121,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); }); } - (void)connman.GetAddedNodeInfo(); + (void)connman.GetAddedNodeInfo(fuzzed_data_provider.ConsumeBool()); (void)connman.GetExtraFullOutboundCount(); (void)connman.GetLocalServices(); (void)connman.GetMaxOutboundTarget(); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 510ee7fb5b..abc230c07d 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -66,7 +66,7 @@ template <typename T, typename P> DataStream Serialize(const T& obj, const P& params) { DataStream ds{}; - ds << WithParams(params, obj); + ds << params(obj); return ds; } @@ -74,7 +74,7 @@ template <typename T, typename P> T Deserialize(DataStream&& ds, const P& params) { T obj; - ds >> WithParams(params, obj); + ds >> params(obj); return obj; } @@ -83,7 +83,7 @@ void DeserializeFromFuzzingInput(FuzzBufferType buffer, T&& obj, const P& params { DataStream ds{buffer}; try { - ds >> WithParams(params, obj); + ds >> params(obj); } catch (const std::ios_base::failure&) { throw invalid_fuzzing_input_exception(); } diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 0534f9bcf1..1f0fa5527a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -14,6 +14,11 @@ /** * Can be used to limit a theoretically unbounded loop. This caps the runtime * to avoid timeouts or OOMs. + * + * This can be used in combination with a check in the condition to confirm + * whether the fuzz engine provided "good" data. If the fuzz input contains + * invalid data, the loop aborts early. This will teach the fuzz engine to look + * for useful data and avoids bloating the fuzz input folder with useless data. */ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 7220c5d997..8d316134cc 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -257,15 +257,6 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) const auto result_package = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit)); - // If something went wrong due to a package-specific policy, it might not return a - // validation result for the transaction. - if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { - auto it = result_package.m_tx_results.find(txs.back()->GetWitnessHash()); - Assert(it != result_package.m_tx_results.end()); - Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - } const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, txs.back(), GetTime(), bypass_limits, /*test_accept=*/!single_submit)); const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; @@ -281,6 +272,12 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) Assert(added.size() == 1); Assert(txs.back() == *added.begin()); } + } else if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { + // We don't know anything about the validity since transactions were randomly generated, so + // just use result_package.m_state here. This makes the expect_valid check meaningless, but + // we can still verify that the contents of m_tx_results are consistent with m_state. + const bool expect_valid{result_package.m_state.IsValid()}; + Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, nullptr)); } else { // This is empty if it fails early checks, or "full" if transactions are looked at deeper Assert(result_package.m_tx_results.size() == txs.size() || result_package.m_tx_results.empty()); diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 227ee9d2c4..4cbc1b4820 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -6,16 +6,14 @@ #include <policy/fees.h> #include <policy/fees_args.h> #include <primitives/transaction.h> +#include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> -#include <txmempool.h> -#include <cstdint> #include <optional> -#include <string> #include <vector> namespace { @@ -31,13 +29,17 @@ void initialize_policy_estimator() FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { + good_data = false; return; } const CTransaction tx{*mtx}; @@ -48,9 +50,11 @@ FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) }, [&] { std::vector<CTxMemPoolEntry> mempool_entries; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { + good_data = false; break; } const CTransaction tx{*mtx}; diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e2..4e059c7d0a 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -3,18 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <base58.h> -#include <core_io.h> #include <key.h> #include <key_io.h> -#include <node/context.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> -#include <rpc/blockchain.h> #include <rpc/client.h> #include <rpc/request.h> #include <rpc/server.h> -#include <rpc/util.h> #include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -22,19 +18,23 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <tinyformat.h> +#include <uint256.h> #include <univalue.h> -#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> +#include <algorithm> +#include <cassert> #include <cstdint> +#include <cstdlib> +#include <exception> #include <iostream> #include <memory> #include <optional> #include <stdexcept> -#include <string> #include <vector> +enum class ChainType; namespace { struct RPCFuzzTestingSetup : public TestingSetup { @@ -184,7 +184,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "waitfornewblock", }; -std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { const size_t max_string_length = 4096; const size_t max_base58_bytes_length{64}; @@ -251,6 +251,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded block std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); if (!opt_block) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; @@ -261,6 +262,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded block header std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!opt_block_header) { + good_data = false; return; } DataStream data_stream{}; @@ -271,6 +273,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded tx std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!opt_tx) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; @@ -281,6 +284,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // base64 encoded psbt std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); if (!opt_psbt) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; @@ -291,6 +295,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // base58 encoded key CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = EncodeSecret(key); @@ -299,6 +304,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded pubkey CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = HexStr(key.GetPubKey()); @@ -306,18 +312,19 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) return r; } -std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { std::vector<std::string> scalar_arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider, good_data)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; } -std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { - return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider, good_data) : ConsumeArrayRPCArgument(fuzzed_data_provider, good_data); } RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() @@ -353,6 +360,7 @@ void initialize_rpc() FUZZ_TARGET(rpc, .init = initialize_rpc) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { @@ -363,8 +371,9 @@ FUZZ_TARGET(rpc, .init = initialize_rpc) return; } std::vector<std::string> arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider, good_data)); } try { rpc_testing_setup->CallRPC(rpc_command, arguments); diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 05b8312ab2..af81fcb593 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -32,7 +32,9 @@ FUZZ_TARGET(socks5, .init = initialize_socks5) ProxyCredentials proxy_credentials; proxy_credentials.username = fuzzed_data_provider.ConsumeRandomLengthString(512); proxy_credentials.password = fuzzed_data_provider.ConsumeRandomLengthString(512); - InterruptSocks5(fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + g_socks5_interrupt(); + } // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 5ec3e89d1e..96095539ec 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -131,6 +131,56 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte return CTxMemPool{mempool_opts}; } +void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, bool wtxid_in_mempool) +{ + + switch (res.m_result_type) { + case MempoolAcceptResult::ResultType::VALID: + { + Assert(txid_in_mempool); + Assert(wtxid_in_mempool); + Assert(res.m_state.IsValid()); + Assert(!res.m_state.IsInvalid()); + Assert(res.m_replaced_transactions); + Assert(res.m_vsize); + Assert(res.m_base_fees); + Assert(res.m_effective_feerate); + Assert(res.m_wtxids_fee_calculations); + Assert(!res.m_other_wtxid); + break; + } + case MempoolAcceptResult::ResultType::INVALID: + { + // It may be already in the mempool since in ATMP cases we don't set MEMPOOL_ENTRY or DIFFERENT_WITNESS + Assert(!res.m_state.IsValid()); + Assert(res.m_state.IsInvalid()); + + const bool is_reconsiderable{res.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + Assert(!res.m_replaced_transactions); + Assert(!res.m_vsize); + Assert(!res.m_base_fees); + // Fee information is provided if the failure is TX_RECONSIDERABLE. + // In other cases, validation may be unable or unwilling to calculate the fees. + Assert(res.m_effective_feerate.has_value() == is_reconsiderable); + Assert(res.m_wtxids_fee_calculations.has_value() == is_reconsiderable); + Assert(!res.m_other_wtxid); + break; + } + case MempoolAcceptResult::ResultType::MEMPOOL_ENTRY: + { + // ATMP never sets this; only set in package settings + Assert(false); + break; + } + case MempoolAcceptResult::ResultType::DIFFERENT_WITNESS: + { + // ATMP never sets this; only set in package settings + Assert(false); + break; + } + } +} + FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -258,9 +308,11 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) SyncWithValidationInterfaceQueue(); UnregisterSharedValidationInterface(txr); + bool txid_in_mempool = tx_pool.exists(GenTxid::Txid(tx->GetHash())); + bool wtxid_in_mempool = tx_pool.exists(GenTxid::Wtxid(tx->GetWitnessHash())); + CheckATMPInvariants(res, txid_in_mempool, wtxid_in_mempool); + Assert(accepted != added.empty()); - Assert(accepted == res.m_state.IsValid()); - Assert(accepted != res.m_state.IsInvalid()); if (accepted) { Assert(added.size() == 1); // For now, no package acceptance Assert(tx == *added.begin()); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 8263cd4c08..95d910b64d 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -107,7 +107,7 @@ template <typename T, typename P> DataStream ds{buffer}; T obj; try { - ds >> WithParams(params, obj); + ds >> params(obj); } catch (const std::ios_base::failure&) { return std::nullopt; } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index 318797faf2..a8f0f3b7ff 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -94,8 +94,8 @@ FUZZ_TARGET(utxo_total_supply) assert(ActiveHeight() == 0); // Get at which height we duplicate the coinbase // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. - // Up to 2000 seems reasonable. - int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Up to 300 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 300); // Always pad with OP_0 at the end to avoid bad-cb-length error const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; // Mine the first block with this duplicate @@ -121,7 +121,7 @@ FUZZ_TARGET(utxo_total_supply) // Limit to avoid timeout, but enough to cover duplicate_coinbase_height // and CVE-2018-17144. - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'00) { CallOneOf( fuzzed_data_provider, diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index db58a0baec..217e4a6d22 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort<descendant_score>(pool, sortedOrder); CTxMemPool::setEntries setAncestors; - setAncestors.insert(pool.mapTx.find(tx6.GetHash())); + setAncestors.insert(pool.GetIter(tx6.GetHash()).value()); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout.resize(1); tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; - setAncestors.insert(pool.mapTx.find(tx7.GetHash())); + setAncestors.insert(pool.GetIter(tx7.GetHash()).value()); pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high @@ -247,8 +247,8 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> snapshotOrder = sortedOrder; - setAncestors.insert(pool.mapTx.find(tx8.GetHash())); - setAncestors.insert(pool.mapTx.find(tx9.GetHash())); + setAncestors.insert(pool.GetIter(tx8.GetHash()).value()); + setAncestors.insert(pool.GetIter(tx9.GetHash()).value()); /* tx10 depends on tx8 and tx9 and has a high fee*/ CMutableTransaction tx10 = CMutableTransaction(); tx10.vin.resize(2); @@ -291,11 +291,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10U); // Now try removing tx10 and verify the sort order returns to normal - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx10.GetHash())), REMOVAL_REASON_DUMMY); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx9.GetHash())), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx8.GetHash())), REMOVAL_REASON_DUMMY); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index f65356936b..311e402e3e 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -15,6 +15,11 @@ BOOST_FIXTURE_TEST_SUITE(miniminer_tests, TestingSetup) +const CAmount low_fee{CENT/2000}; // 500 ṩ +const CAmount med_fee{CENT/200}; // 5000 ṩ +const CAmount high_fee{CENT/10}; // 100_000 ṩ + + static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, size_t num_outputs) { CMutableTransaction tx = CMutableTransaction(); @@ -67,21 +72,51 @@ Value Find(const std::map<Key, Value>& map, const Key& key) return it->second; } -BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(miniminer_negative, TestChain100Setup) { CTxMemPool& pool = *Assert(m_node.mempool); LOCK2(::cs_main, pool.cs); TestMemPoolEntryHelper entry; - const CAmount low_fee{CENT/2000}; - const CAmount normal_fee{CENT/200}; - const CAmount high_fee{CENT/10}; + // Create a transaction that will be prioritised to have a negative modified fee. + const CAmount positive_base_fee{1000}; + const CAmount negative_fee_delta{-50000}; + const CAmount negative_modified_fees{positive_base_fee + negative_fee_delta}; + BOOST_CHECK(negative_modified_fees < 0); + const auto tx_mod_negative = make_tx({COutPoint{m_coinbase_txns[4]->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(positive_base_fee).FromTx(tx_mod_negative)); + pool.PrioritiseTransaction(tx_mod_negative->GetHash(), negative_fee_delta); + const COutPoint only_outpoint{tx_mod_negative->GetHash(), 0}; + + // When target feerate is 0, transactions with negative fees are not selected. + node::MiniMiner mini_miner_target0(pool, {only_outpoint}); + BOOST_CHECK(mini_miner_target0.IsReadyToCalculate()); + const CFeeRate feerate_zero(0); + mini_miner_target0.BuildMockTemplate(feerate_zero); + // Check the quit condition: + BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(Assert(pool.GetEntry(tx_mod_negative->GetHash()))->GetTxSize())); + BOOST_CHECK(mini_miner_target0.GetMockTemplateTxids().empty()); + + // With no target feerate, the template includes all transactions, even negative feerate ones. + node::MiniMiner mini_miner_no_target(pool, {only_outpoint}); + BOOST_CHECK(mini_miner_no_target.IsReadyToCalculate()); + mini_miner_no_target.BuildMockTemplate(std::nullopt); + const auto template_txids{mini_miner_no_target.GetMockTemplateTxids()}; + BOOST_CHECK_EQUAL(template_txids.size(), 1); + BOOST_CHECK(template_txids.count(tx_mod_negative->GetHash().ToUint256()) > 0); +} + +BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; // Create a parent tx0 and child tx1 with normal fees: const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx0)); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx0)); const auto tx1 = make_tx({COutPoint{tx0->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx1)); // Create a low-feerate parent tx2 and high-feerate child tx3 (cpfp) const auto tx2 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); @@ -94,9 +129,11 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) pool.addUnchecked(entry.Fee(low_fee).FromTx(tx4)); const auto tx5 = make_tx({COutPoint{tx4->GetHash(), 0}}, /*num_outputs=*/1); pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + const CAmount tx5_delta{CENT/100}; // Make tx5's modified fee much higher than its base fee. This should cause it to pass // the fee-related checks despite being low-feerate. - pool.PrioritiseTransaction(tx5->GetHash(), CENT/100); + pool.PrioritiseTransaction(tx5->GetHash(), tx5_delta); + const CAmount tx5_mod_fee{low_fee + tx5_delta}; // Create a high-feerate parent tx6, low-feerate child tx7 const auto tx6 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); @@ -142,9 +179,9 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) }; std::map<uint256, TxDimensions> tx_dims; for (const auto& tx : all_transactions) { - const auto it = pool.GetIter(tx->GetHash()).value(); - tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), - CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + const auto& entry{*Assert(pool.GetEntry(tx->GetHash()))}; + tx_dims.emplace(tx->GetHash(), TxDimensions{entry.GetTxSize(), entry.GetModifiedFee(), + CFeeRate(entry.GetModifiedFee(), entry.GetTxSize())}); } const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), @@ -273,6 +310,64 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) } } } + + // Check m_inclusion_order for equivalent mempool- and manually-constructed MiniMiners. + // (We cannot check bump fees in manually-constructed MiniMiners because it doesn't know what + // outpoints are requested). + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + { + const int32_t tx0_vsize{tx_dims.at(tx0->GetHash()).vsize}; + const int32_t tx1_vsize{tx_dims.at(tx1->GetHash()).vsize}; + const int32_t tx2_vsize{tx_dims.at(tx2->GetHash()).vsize}; + const int32_t tx3_vsize{tx_dims.at(tx3->GetHash()).vsize}; + const int32_t tx4_vsize{tx_dims.at(tx4->GetHash()).vsize}; + const int32_t tx5_vsize{tx_dims.at(tx5->GetHash()).vsize}; + const int32_t tx6_vsize{tx_dims.at(tx6->GetHash()).vsize}; + const int32_t tx7_vsize{tx_dims.at(tx7->GetHash()).vsize}; + + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize,/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee); + miniminer_info.emplace_back(tx1, tx1_vsize, tx0_vsize + tx1_vsize, med_fee, 2*med_fee); + miniminer_info.emplace_back(tx2, tx2_vsize, tx2_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx3, tx3_vsize, tx2_vsize + tx3_vsize, high_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx4, tx4_vsize, tx4_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx5, tx5_vsize, tx4_vsize + tx5_vsize, tx5_mod_fee, low_fee + tx5_mod_fee); + miniminer_info.emplace_back(tx6, tx6_vsize, tx6_vsize, high_fee, high_fee); + miniminer_info.emplace_back(tx7, tx7_vsize, tx6_vsize + tx7_vsize, low_fee, high_fee + low_fee); + } + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx1->GetHash()}); + descendant_caches.emplace(tx1->GetHash(), std::set<Txid>{tx1->GetHash()}); + descendant_caches.emplace(tx2->GetHash(), std::set<Txid>{tx2->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx3->GetHash(), std::set<Txid>{tx3->GetHash()}); + descendant_caches.emplace(tx4->GetHash(), std::set<Txid>{tx4->GetHash(), tx5->GetHash()}); + descendant_caches.emplace(tx5->GetHash(), std::set<Txid>{tx5->GetHash()}); + descendant_caches.emplace(tx6->GetHash(), std::set<Txid>{tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx7->GetHash(), std::set<Txid>{tx7->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + // Use unspent outpoints to avoid entries being omitted. + node::MiniMiner miniminer_pool(pool, all_unspent_outpoints); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + BOOST_CHECK(miniminer_pool.IsReadyToCalculate()); + for (const auto& sequences : {miniminer_manual.Linearize(), miniminer_pool.Linearize()}) { + // tx6 is selected first: high feerate with no parents to bump + BOOST_CHECK_EQUAL(Find(sequences, tx6->GetHash()), 0); + + // tx2 + tx3 CPFP are selected next + BOOST_CHECK_EQUAL(Find(sequences, tx2->GetHash()), 1); + BOOST_CHECK_EQUAL(Find(sequences, tx3->GetHash()), 1); + + // tx4 + prioritised tx5 CPFP + BOOST_CHECK_EQUAL(Find(sequences, tx4->GetHash()), 2); + BOOST_CHECK_EQUAL(Find(sequences, tx5->GetHash()), 2); + + BOOST_CHECK_EQUAL(Find(sequences, tx0->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx1->GetHash()), 3); + + + // tx7 is selected last: low feerate with no children + BOOST_CHECK_EQUAL(Find(sequences, tx7->GetHash()), 4); + } } BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) @@ -308,10 +403,6 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) LOCK2(::cs_main, pool.cs); TestMemPoolEntryHelper entry; - const CAmount low_fee{CENT/2000}; // 500 ṩ - const CAmount med_fee{CENT/200}; // 5000 ṩ - const CAmount high_fee{CENT/10}; // 100_000 ṩ - // Create 3 parents of different feerates, and 1 child spending outputs from all 3 parents. const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); pool.addUnchecked(entry.Fee(low_fee).FromTx(tx0)); @@ -356,15 +447,15 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) // tx3's feerate is lower than tx2's. same fee, different weight. BOOST_CHECK(tx2_feerate > tx3_feerate); const auto tx3_anc_feerate = CFeeRate(low_fee + med_fee + high_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]); - const auto tx3_iter = pool.GetIter(tx3->GetHash()); - BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_iter.value()->GetModFeesWithAncestors(), tx3_iter.value()->GetSizeWithAncestors())); + const auto& tx3_entry{*Assert(pool.GetEntry(tx3->GetHash()))}; + BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_entry.GetModFeesWithAncestors(), tx3_entry.GetSizeWithAncestors())); const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[4]); const auto tx6_anc_feerate = CFeeRate(high_fee + low_fee + med_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]); - const auto tx6_iter = pool.GetIter(tx6->GetHash()); - BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_iter.value()->GetModFeesWithAncestors(), tx6_iter.value()->GetSizeWithAncestors())); + const auto& tx6_entry{*Assert(pool.GetEntry(tx6->GetHash()))}; + BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_entry.GetModFeesWithAncestors(), tx6_entry.GetSizeWithAncestors())); const auto tx7_anc_feerate = CFeeRate(high_fee + low_fee + high_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]); - const auto tx7_iter = pool.GetIter(tx7->GetHash()); - BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_iter.value()->GetModFeesWithAncestors(), tx7_iter.value()->GetSizeWithAncestors())); + const auto& tx7_entry{*Assert(pool.GetEntry(tx7->GetHash()))}; + BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_entry.GetModFeesWithAncestors(), tx7_entry.GetSizeWithAncestors())); BOOST_CHECK(tx4_feerate > tx6_anc_feerate); BOOST_CHECK(tx4_feerate > tx7_anc_feerate); @@ -450,6 +541,49 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) BOOST_CHECK(tx7_bumpfee != bump_fees.end()); BOOST_CHECK_EQUAL(tx7_bumpfee->second, 0); } + // Check linearization order + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], /*fee_self=*/low_fee, /*fee_ancestor=*/low_fee); + miniminer_info.emplace_back(tx1, tx_vsizes[1], tx_vsizes[1], med_fee, med_fee); + miniminer_info.emplace_back(tx2, tx_vsizes[2], tx_vsizes[2], high_fee, high_fee); + miniminer_info.emplace_back(tx3, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], high_fee, low_fee+med_fee+2*high_fee); + miniminer_info.emplace_back(tx4, tx_vsizes[4], tx_vsizes[4], high_fee, high_fee); + miniminer_info.emplace_back(tx5, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], low_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx6, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], med_fee, high_fee+low_fee+med_fee); + miniminer_info.emplace_back(tx7, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], high_fee, high_fee+low_fee+high_fee); + + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx1->GetHash(), std::set<Txid>{tx1->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx2->GetHash(), std::set<Txid>{tx2->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx3->GetHash(), std::set<Txid>{tx3->GetHash()}); + descendant_caches.emplace(tx4->GetHash(), std::set<Txid>{tx4->GetHash(), tx5->GetHash(), tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx5->GetHash(), std::set<Txid>{tx5->GetHash(), tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx6->GetHash(), std::set<Txid>{tx6->GetHash()}); + descendant_caches.emplace(tx7->GetHash(), std::set<Txid>{tx7->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + // Use unspent outpoints to avoid entries being omitted. + node::MiniMiner miniminer_pool(pool, all_unspent_outpoints); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + BOOST_CHECK(miniminer_pool.IsReadyToCalculate()); + for (const auto& sequences : {miniminer_manual.Linearize(), miniminer_pool.Linearize()}) { + // tx2 and tx4 selected first: high feerate with nothing to bump + BOOST_CHECK_EQUAL(Find(sequences, tx4->GetHash()), 0); + BOOST_CHECK_EQUAL(Find(sequences, tx2->GetHash()), 1); + + // tx5 + tx7 CPFP + BOOST_CHECK_EQUAL(Find(sequences, tx5->GetHash()), 2); + BOOST_CHECK_EQUAL(Find(sequences, tx7->GetHash()), 2); + + // tx0 and tx1 CPFP'd by tx3 + BOOST_CHECK_EQUAL(Find(sequences, tx0->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx1->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx3->GetHash()), 3); + + // tx6 at medium feerate + BOOST_CHECK_EQUAL(Find(sequences, tx6->GetHash()), 4); + } } BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) { @@ -507,4 +641,64 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) } } +BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(cs_main, pool.cs); + { + // 3 pairs of grandparent + fee-bumping parent, plus 1 low-feerate child. + // 0 fee + high fee + auto grandparent_zero_fee = make_tx({{m_coinbase_txns.at(0)->GetHash(), 0}}, 1); + auto parent_high_feerate = make_tx({{grandparent_zero_fee->GetHash(), 0}}, 1); + // double low fee + med fee + auto grandparent_double_low_feerate = make_tx({{m_coinbase_txns.at(2)->GetHash(), 0}}, 1); + auto parent_med_feerate = make_tx({{grandparent_double_low_feerate->GetHash(), 0}}, 1); + // low fee + double low fee + auto grandparent_low_feerate = make_tx({{m_coinbase_txns.at(1)->GetHash(), 0}}, 1); + auto parent_double_low_feerate = make_tx({{grandparent_low_feerate->GetHash(), 0}}, 1); + // child is below the cpfp package feerates because it is larger than everything else + auto child = make_tx({{parent_high_feerate->GetHash(), 0}, {parent_double_low_feerate->GetHash(), 0}, {parent_med_feerate->GetHash(), 0}}, 1); + + // We artificially record each transaction (except the child) with a uniform vsize of 100vB. + const int64_t tx_vsize{100}; + const int64_t child_vsize{1000}; + + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + miniminer_info.emplace_back(grandparent_zero_fee, /*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, /*fee_self=*/0,/*fee_ancestor=*/0); + miniminer_info.emplace_back(parent_high_feerate, tx_vsize, 2*tx_vsize, high_fee, high_fee); + miniminer_info.emplace_back(grandparent_double_low_feerate, tx_vsize, tx_vsize, 2*low_fee, 2*low_fee); + miniminer_info.emplace_back(parent_med_feerate, tx_vsize, 2*tx_vsize, med_fee, 2*low_fee+med_fee); + miniminer_info.emplace_back(grandparent_low_feerate, tx_vsize, tx_vsize, low_fee, low_fee); + miniminer_info.emplace_back(parent_double_low_feerate, tx_vsize, 2*tx_vsize, 2*low_fee, 3*low_fee); + miniminer_info.emplace_back(child, child_vsize, 6*tx_vsize+child_vsize, low_fee, high_fee+med_fee+6*low_fee); + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(grandparent_zero_fee->GetHash(), std::set<Txid>{grandparent_zero_fee->GetHash(), parent_high_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(grandparent_low_feerate->GetHash(), std::set<Txid>{grandparent_low_feerate->GetHash(), parent_double_low_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(grandparent_double_low_feerate->GetHash(), std::set<Txid>{grandparent_double_low_feerate->GetHash(), parent_med_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_high_feerate->GetHash(), std::set<Txid>{parent_high_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_med_feerate->GetHash(), std::set<Txid>{parent_med_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_double_low_feerate->GetHash(), std::set<Txid>{parent_double_low_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(child->GetHash(), std::set<Txid>{child->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + const auto sequences{miniminer_manual.Linearize()}; + + // CPFP zero + high + BOOST_CHECK_EQUAL(sequences.at(grandparent_zero_fee->GetHash()), 0); + BOOST_CHECK_EQUAL(sequences.at(parent_high_feerate->GetHash()), 0); + + // CPFP double low + med + BOOST_CHECK_EQUAL(sequences.at(grandparent_double_low_feerate->GetHash()), 1); + BOOST_CHECK_EQUAL(sequences.at(parent_med_feerate->GetHash()), 1); + + // CPFP low + double low + BOOST_CHECK_EQUAL(sequences.at(grandparent_low_feerate->GetHash()), 2); + BOOST_CHECK_EQUAL(sequences.at(parent_double_low_feerate->GetHash()), 2); + + // Child at the end + BOOST_CHECK_EQUAL(sequences.at(child->GetHash()), 3); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp new file mode 100644 index 0000000000..3d3f296d82 --- /dev/null +++ b/src/test/net_peer_connection_tests.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compat/compat.h> +#include <net.h> +#include <net_processing.h> +#include <netaddress.h> +#include <netbase.h> +#include <netgroup.h> +#include <node/connection_types.h> +#include <protocol.h> +#include <random.h> +#include <test/util/logging.h> +#include <test/util/net.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <util/chaintype.h> +#include <version.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <boost/test/unit_test.hpp> + +struct LogIPsTestingSetup : public TestingSetup { + LogIPsTestingSetup() + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-logips"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(net_peer_connection_tests, LogIPsTestingSetup) + +static CService ip(uint32_t i) +{ + struct in_addr s; + s.s_addr = i; + return CService{CNetAddr{s}, Params().GetDefaultPort()}; +} + +/** Create a peer and connect to it. If the optional `address` (IP/CJDNS only) isn't passed, a random address is created. */ +static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman, ConnmanTestMsg& connman, ConnectionType conn_type, bool onion_peer = false, std::optional<std::string> address = std::nullopt) +{ + CAddress addr{}; + + if (address.has_value()) { + addr = CAddress{MaybeFlipIPv6toCJDNS(LookupNumeric(address.value(), Params().GetDefaultPort())), NODE_NONE}; + } else if (onion_peer) { + auto tor_addr{g_insecure_rand_ctx.randbytes(ADDR_TORV3_SIZE)}; + BOOST_REQUIRE(addr.SetSpecial(OnionToString(tor_addr))); + } + + while (!addr.IsLocal() && !addr.IsRoutable()) { + addr = CAddress{ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE}; + } + + BOOST_REQUIRE(addr.IsValid()); + + const bool inbound_onion{onion_peer && conn_type == ConnectionType::INBOUND}; + + nodes.emplace_back(new CNode{++id, + /*sock=*/nullptr, + addr, + /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, + CAddress{}, + /*addrNameIn=*/"", + conn_type, + /*inbound_onion=*/inbound_onion}); + CNode& node = *nodes.back(); + node.SetCommonVersion(PROTOCOL_VERSION); + + peerman.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node.fSuccessfullyConnected = true; + + connman.AddTestNode(node); +} + +BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) +{ + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + NodeId id{0}; + std::vector<CNode*> nodes; + + // Connect a localhost peer. + { + ASSERT_DEBUG_LOG("Added connection to 127.0.0.1:8333 peer=1"); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::MANUAL, /*onion_peer=*/false, /*address=*/"127.0.0.1"); + BOOST_REQUIRE(nodes.back() != nullptr); + } + + // Call ConnectNode(), which is also called by RPC addnode onetry, for a localhost + // address that resolves to multiple IPs, including that of the connected peer. + // The connection attempt should consistently fail due to the check in ConnectNode(). + for (int i = 0; i < 10; ++i) { + ASSERT_DEBUG_LOG("Not opening a connection to localhost, already connected to 127.0.0.1:8333"); + BOOST_CHECK(!connman->ConnectNodePublic(*peerman, "localhost", ConnectionType::MANUAL)); + } + + // Add 3 more peer connections. + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::BLOCK_RELAY, /*onion_peer=*/true); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND); + + BOOST_TEST_MESSAGE("Call AddNode() for all the peers"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddNode({/*m_added_node=*/node->addr.ToStringAddrPort(), /*m_use_v2transport=*/true})); + BOOST_TEST_MESSAGE(strprintf("peer id=%s addr=%s", node->GetId(), node->addr.ToStringAddrPort())); + } + + BOOST_TEST_MESSAGE("\nCall AddNode() with 2 addrs resolving to existing localhost addnode entry; neither should be added"); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.0.0.1", /*m_use_v2transport=*/true})); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.1", /*m_use_v2transport=*/true})); + + BOOST_TEST_MESSAGE("\nExpect GetAddedNodeInfo to return expected number of peers with `include_connected` true/false"); + BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); + BOOST_CHECK(connman->GetAddedNodeInfo(/*include_connected=*/false).empty()); + + BOOST_TEST_MESSAGE("\nPrint GetAddedNodeInfo contents:"); + for (const auto& info : connman->GetAddedNodeInfo(/*include_connected=*/true)) { + BOOST_TEST_MESSAGE(strprintf("\nadded node: %s", info.m_params.m_added_node)); + BOOST_TEST_MESSAGE(strprintf("connected: %s", info.fConnected)); + if (info.fConnected) { + BOOST_TEST_MESSAGE(strprintf("IP address: %s", info.resolvedAddress.ToStringAddrPort())); + BOOST_TEST_MESSAGE(strprintf("direction: %s", info.fInbound ? "inbound" : "outbound")); + } + } + + BOOST_TEST_MESSAGE("\nCheck that all connected peers are correctly detected as connected"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AlreadyConnectedPublic(node->addr)); + } + + // Clean up + for (auto node : connman->TestNodes()) { + peerman->FinalizeNode(*node); + } + connman->ClearTestNodes(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 7c98c382e4..48e0706a53 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -331,17 +331,17 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) DataStream s{}; const auto ser_params{CAddress::V1_NETWORK}; - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); addr = LookupHost("1.2.3.4", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); s.clear(); addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -349,12 +349,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) BOOST_CHECK(!addr.SetSpecial("6hzph5hv6337r6p2.onion")); BOOST_REQUIRE(addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); addr.SetInternal("a"); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); s.clear(); } @@ -365,17 +365,17 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) DataStream s{}; const auto ser_params{CAddress::V2_NETWORK}; - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); s.clear(); addr = LookupHost("1.2.3.4", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); s.clear(); addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -383,12 +383,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) BOOST_CHECK(!addr.SetSpecial("6hzph5hv6337r6p2.onion")); BOOST_REQUIRE(addr.SetSpecial("kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "042053cd5648488c4707914182655b7664034e09e66f7e8cbf1084e654eb56c5bd88"); s.clear(); BOOST_REQUIRE(addr.SetInternal("a")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "0210fd6b88c08724ca978112ca1bbdcafac2"); s.clear(); } @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "04" // address length "01020304")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv4()); BOOST_CHECK(addr.IsAddrV1Compatible()); @@ -414,7 +414,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "04" // address length "0102")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, HasReason("end of data")); + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("end of data")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -422,7 +422,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "05" // address length "01020304")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 IPv4 address with length 5 (should be 4)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -431,7 +431,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "fd0102" // address length (513 as CompactSize) "01020304")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("Address too long: 513 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -440,7 +440,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "0102030405060708090a0b0c0d0e0f10")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv6()); BOOST_CHECK(addr.IsAddrV1Compatible()); @@ -453,7 +453,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "fd6b88c08724ca978112ca1bbdcafac2")}; // address: 0xfd + sha256("bitcoin")[0:5] + // sha256(name)[0:10] - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsInternal()); BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToStringAddr(), "zklycewkdo64v6wc.internal"); @@ -463,7 +463,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "04" // address length "00")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 IPv6 address with length 4 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -472,7 +472,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "00000000000000000000ffff01020304")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -480,7 +480,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "fd87d87eeb430102030405060708090a")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -488,7 +488,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("03" // network type (TORv2) "0a" // address length "f1f2f3f4f5f6f7f8f9fa")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -498,7 +498,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "79bcc625184b05194975c28b66b66b04" // address "69f7f6556fb1ac3189a79b40dda32f1f" )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsTor()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -511,7 +511,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "00" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 TORv3 address with length 0 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -521,7 +521,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "20" // address length "a2894dabaec08c0051a481a6dac88b64" // address "f98232ae42d4b6fd2fa81952dfe36a87")}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsI2P()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -534,7 +534,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "03" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 I2P address with length 3 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -544,7 +544,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "fc000001000200030004000500060007" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsCJDNS()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -556,7 +556,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "aa000001000200030004000500060007" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsCJDNS()); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -566,7 +566,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "01" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 CJDNS address with length 1 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -576,7 +576,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "fe00000002" // address length (CompactSize's MAX_SIZE) "01020304050607" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("Address too long: 33554432 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -586,7 +586,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "04" // address length "01020304" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -595,7 +595,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "00" // address length "" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); } diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index d18d2623b1..49827be559 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -255,10 +255,15 @@ BOOST_AUTO_TEST_CASE(class_methods) } } -enum class BaseFormat { - RAW, - HEX, +struct BaseFormat { + const enum { + RAW, + HEX, + } m_base_format; + SER_PARAMS_OPFUNC }; +constexpr BaseFormat RAW{BaseFormat::RAW}; +constexpr BaseFormat HEX{BaseFormat::HEX}; /// (Un)serialize a number as raw byte or 2 hexadecimal chars. class Base @@ -272,7 +277,7 @@ public: template <typename Stream> void Serialize(Stream& s) const { - if (s.GetParams() == BaseFormat::RAW) { + if (s.GetParams().m_base_format == BaseFormat::RAW) { s << m_base_data; } else { s << Span{HexStr(Span{&m_base_data, 1})}; @@ -282,7 +287,7 @@ public: template <typename Stream> void Unserialize(Stream& s) { - if (s.GetParams() == BaseFormat::RAW) { + if (s.GetParams().m_base_format == BaseFormat::RAW) { s >> m_base_data; } else { std::string hex{"aa"}; @@ -301,6 +306,8 @@ public: LOWER, UPPER, } m_derived_format; + + SER_PARAMS_OPFUNC }; class Derived : public Base @@ -310,7 +317,7 @@ public: SERIALIZE_METHODS_PARAMS(Derived, obj, DerivedAndBaseFormat, fmt) { - READWRITE(WithParams(fmt.m_base_format, AsBase<Base>(obj))); + READWRITE(fmt.m_base_format(AsBase<Base>(obj))); if (ser_action.ForRead()) { std::string str; @@ -330,20 +337,20 @@ BOOST_AUTO_TEST_CASE(with_params_base) DataStream stream; - stream << WithParams(BaseFormat::RAW, b); + stream << RAW(b); BOOST_CHECK_EQUAL(stream.str(), "\x0F"); b.m_base_data = 0; - stream >> WithParams(BaseFormat::RAW, b); + stream >> RAW(b); BOOST_CHECK_EQUAL(b.m_base_data, 0x0F); stream.clear(); - stream << WithParams(BaseFormat::HEX, b); + stream << HEX(b); BOOST_CHECK_EQUAL(stream.str(), "0f"); b.m_base_data = 0; - stream >> WithParams(BaseFormat::HEX, b); + stream >> HEX(b); BOOST_CHECK_EQUAL(b.m_base_data, 0x0F); } @@ -353,45 +360,42 @@ BOOST_AUTO_TEST_CASE(with_params_vector_of_base) DataStream stream; - stream << WithParams(BaseFormat::RAW, v); + stream << RAW(v); BOOST_CHECK_EQUAL(stream.str(), "\x02\x0F\xFF"); v[0].m_base_data = 0; v[1].m_base_data = 0; - stream >> WithParams(BaseFormat::RAW, v); + stream >> RAW(v); BOOST_CHECK_EQUAL(v[0].m_base_data, 0x0F); BOOST_CHECK_EQUAL(v[1].m_base_data, 0xFF); stream.clear(); - stream << WithParams(BaseFormat::HEX, v); + stream << HEX(v); BOOST_CHECK_EQUAL(stream.str(), "\x02" "0fff"); v[0].m_base_data = 0; v[1].m_base_data = 0; - stream >> WithParams(BaseFormat::HEX, v); + stream >> HEX(v); BOOST_CHECK_EQUAL(v[0].m_base_data, 0x0F); BOOST_CHECK_EQUAL(v[1].m_base_data, 0xFF); } +constexpr DerivedAndBaseFormat RAW_LOWER{{BaseFormat::RAW}, DerivedAndBaseFormat::DerivedFormat::LOWER}; +constexpr DerivedAndBaseFormat HEX_UPPER{{BaseFormat::HEX}, DerivedAndBaseFormat::DerivedFormat::UPPER}; + BOOST_AUTO_TEST_CASE(with_params_derived) { Derived d; d.m_base_data = 0x0F; d.m_derived_data = "xY"; - DerivedAndBaseFormat fmt; - DataStream stream; - fmt.m_base_format = BaseFormat::RAW; - fmt.m_derived_format = DerivedAndBaseFormat::DerivedFormat::LOWER; - stream << WithParams(fmt, d); + stream << RAW_LOWER(d); - fmt.m_base_format = BaseFormat::HEX; - fmt.m_derived_format = DerivedAndBaseFormat::DerivedFormat::UPPER; - stream << WithParams(fmt, d); + stream << HEX_UPPER(d); BOOST_CHECK_EQUAL(stream.str(), "\x0F\x02xy" "0f\x02XY"); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 4d9a5ef7f3..84c9ecc3d1 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -9,7 +9,9 @@ #include <primitives/transaction.h> #include <script/script.h> #include <test/util/random.h> +#include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -47,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) package_too_many.emplace_back(create_placeholder_tx(1, 1)); } PackageValidationState state_too_many; - BOOST_CHECK(!CheckPackage(package_too_many, state_too_many)); + BOOST_CHECK(!IsWellFormedPackage(package_too_many, state_too_many, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_too_many.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_too_many.GetRejectReason(), "package-too-many-transactions"); @@ -62,7 +64,7 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) } BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); PackageValidationState state_too_large; - BOOST_CHECK(!CheckPackage(package_too_large, state_too_large)); + BOOST_CHECK(!IsWellFormedPackage(package_too_large, state_too_large, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_too_large.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_too_large.GetRejectReason(), "package-too-large"); @@ -73,9 +75,39 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) package_duplicate_txids_empty.emplace_back(MakeTransactionRef(empty_tx)); } PackageValidationState state_duplicates; - BOOST_CHECK(!CheckPackage(package_duplicate_txids_empty, state_duplicates)); + BOOST_CHECK(!IsWellFormedPackage(package_duplicate_txids_empty, state_duplicates, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_duplicates.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_duplicates.GetRejectReason(), "package-contains-duplicates"); + BOOST_CHECK(!IsConsistentPackage(package_duplicate_txids_empty)); + + // Packages can't have transactions spending the same prevout + CMutableTransaction tx_zero_1; + CMutableTransaction tx_zero_2; + COutPoint same_prevout{InsecureRand256(), 0}; + tx_zero_1.vin.emplace_back(same_prevout); + tx_zero_2.vin.emplace_back(same_prevout); + // Different vouts (not the same tx) + tx_zero_1.vout.emplace_back(CENT, P2WSH_OP_TRUE); + tx_zero_2.vout.emplace_back(2 * CENT, P2WSH_OP_TRUE); + Package package_conflicts{MakeTransactionRef(tx_zero_1), MakeTransactionRef(tx_zero_2)}; + BOOST_CHECK(!IsConsistentPackage(package_conflicts)); + // Transactions are considered sorted when they have no dependencies. + BOOST_CHECK(IsTopoSortedPackage(package_conflicts)); + PackageValidationState state_conflicts; + BOOST_CHECK(!IsWellFormedPackage(package_conflicts, state_conflicts, /*require_sorted=*/true)); + BOOST_CHECK_EQUAL(state_conflicts.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_conflicts.GetRejectReason(), "conflict-in-package"); + + // IsConsistentPackage only cares about conflicts between transactions, not about a transaction + // conflicting with itself (i.e. duplicate prevouts in vin). + CMutableTransaction dup_tx; + const COutPoint rand_prevout{InsecureRand256(), 0}; + dup_tx.vin.emplace_back(rand_prevout); + dup_tx.vin.emplace_back(rand_prevout); + Package package_with_dup_tx{MakeTransactionRef(dup_tx)}; + BOOST_CHECK(IsConsistentPackage(package_with_dup_tx)); + package_with_dup_tx.emplace_back(create_placeholder_tx(1, 1)); + BOOST_CHECK(IsConsistentPackage(package_with_dup_tx)); } BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) @@ -101,36 +133,35 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_destination=*/child_locking_script, /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - BOOST_CHECK(result_parent_child.m_tx_results.size() == 2); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + Package package_parent_child{tx_parent, tx_child}; + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + + BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); + BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + } // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /*test_accept=*/true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - BOOST_CHECK(result_single_large.m_tx_results.size() == 1); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + Package package_single_giant{giant_ptx}; + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true); + if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { + BOOST_ERROR(err_single_large.value()); + } else { + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + } // Check that mempool size hasn't changed. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); @@ -157,8 +188,8 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) CTransactionRef tx_child = MakeTransactionRef(mtx_child); PackageValidationState state; - BOOST_CHECK(CheckPackage({tx_parent, tx_child}, state)); - BOOST_CHECK(!CheckPackage({tx_child, tx_parent}, state)); + BOOST_CHECK(IsWellFormedPackage({tx_parent, tx_child}, state, /*require_sorted=*/true)); + BOOST_CHECK(!IsWellFormedPackage({tx_child, tx_parent}, state, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state.GetRejectReason(), "package-not-sorted"); BOOST_CHECK(IsChildWithParents({tx_parent, tx_child})); @@ -186,7 +217,7 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) package.push_back(MakeTransactionRef(child)); PackageValidationState state; - BOOST_CHECK(CheckPackage(package, state)); + BOOST_CHECK(IsWellFormedPackage(package, state, /*require_sorted=*/true)); BOOST_CHECK(IsChildWithParents(package)); BOOST_CHECK(IsChildWithParentsTree(package)); @@ -224,8 +255,8 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) BOOST_CHECK(!IsChildWithParentsTree({tx_parent, tx_parent_also_child, tx_child})); // IsChildWithParents does not detect unsorted parents. BOOST_CHECK(IsChildWithParents({tx_parent_also_child, tx_parent, tx_child})); - BOOST_CHECK(CheckPackage({tx_parent, tx_parent_also_child, tx_child}, state)); - BOOST_CHECK(!CheckPackage({tx_parent_also_child, tx_parent, tx_child}, state)); + BOOST_CHECK(IsWellFormedPackage({tx_parent, tx_parent_also_child, tx_child}, state, /*require_sorted=*/true)); + BOOST_CHECK(!IsWellFormedPackage({tx_parent_also_child, tx_parent, tx_child}, state, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state.GetRejectReason(), "package-not-sorted"); } @@ -250,6 +281,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_unrelated, /*test_accept=*/false); + // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -305,20 +337,20 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CMutableTransaction mtx_parent_invalid{mtx_parent}; mtx_parent_invalid.vin[0].scriptWitness = bad_witness; CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); + Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {tx_parent_invalid, tx_child}, /*test_accept=*/ false); - BOOST_CHECK(result_quit_early.m_state.IsInvalid()); + package_invalid_parent, /*test_accept=*/ false); + if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_parent_invalid.value()); + } else { + auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); + auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); + } BOOST_CHECK_EQUAL(result_quit_early.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(!result_quit_early.m_tx_results.empty()); - BOOST_CHECK_EQUAL(result_quit_early.m_tx_results.size(), 2); - auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); - auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_quit_early.m_tx_results.end()); - BOOST_CHECK(it_child != result_quit_early.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); } // Child with missing parent. @@ -350,36 +382,27 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_parent))); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != submit_parent_child.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_state.IsValid()); BOOST_CHECK(it_child->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_child))); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_deduped.m_tx_results.size(), package_parent_child.size()); - auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_parent_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_child_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_deduped.value()); + } else { + auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } } @@ -439,51 +462,39 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are // same-txid-different-witness. { + Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness1.m_tx_results.size(), 2); - auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child1 = submit_witness1.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_parent1->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_child1->second.m_state.GetRejectReason()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + package_parent_child1, /*test_accept=*/false); + if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness1.value()); + } // Child2 would have been validated individually. + Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness2.m_tx_results.size(), 2); - auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); - BOOST_CHECK(it_parent2_deduped != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child2 != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + package_parent_child2, /*test_accept=*/false); + if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness2.value()); + } else { + auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); + BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); + } // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_segwit_dedup.m_tx_results.size(), 2); - auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + package_parent_child1, /*test_accept=*/false); + if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_segwit_dedup.value()); + } else { + auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); + BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } } // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as @@ -504,21 +515,17 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // We already submitted child1 above. { + Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_spend_ignored.m_tx_results.size(), 2); - auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); - auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); - BOOST_CHECK(it_child2_ignored != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_grandchild != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + package_child2_grandchild, /*test_accept=*/false); + if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_spend_ignored.value()); + } else { + auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); + auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); + BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + } } // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of @@ -610,36 +617,28 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // child should be accepted { const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); - BOOST_CHECK_MESSAGE(mixed_result.m_state.IsValid(), mixed_result.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(mixed_result.m_tx_results.size(), package_mixed.size()); - auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); - auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); - auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); - auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); - BOOST_CHECK(it_parent1 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent2 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent3 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_child != mixed_result.m_tx_results.end()); - - BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent1->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent2_v1->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); - - // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. - const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); - BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); - BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_mixed.value()); + } else { + auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); + auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); + auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); + auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); + + BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); + + // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. + const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); + BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); + BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } } } @@ -684,15 +683,17 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(submit_cpfp_deprio.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - const CFeeRate expected_feerate(0, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp_deprio.value()); + } else { + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } } // Clear the prioritisation of the parent transaction. @@ -704,31 +705,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); + if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp.value()); + } else { + auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); + + const CFeeRate expected_feerate(coinbase_value - child_value, + GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_cpfp.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_cpfp.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_cpfp.m_tx_results.size(), package_cpfp.size()); - auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); - BOOST_CHECK(it_child != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); - - const CFeeRate expected_feerate(coinbase_value - child_value, - GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); } // Just because we allow low-fee parents doesn't mean we allow low-feerate packages. @@ -754,15 +751,28 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) package_still_too_low.push_back(tx_child_cheap); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_child_cheap)) <= child_fee); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)) > parent_fee + child_fee); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - // Cheap package should fail with package-fee-too-low. + // Cheap package should fail for being too low fee. { - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_package_too_low.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "package-fee-too-low"); + if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_package_too_low.value()); + } else { + // Individual feerate of parent is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee, GetVirtualTransactionSize(*tx_parent_cheap))); + // Package feerate of parent + child is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee + child_fee, GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap))); + } + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "transaction failed"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } @@ -773,25 +783,26 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); + if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_prioritised.value()); + } else { + const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); + auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); + auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_prioritised_package.m_state.IsValid(), - "Package validation unexpectedly failed" << submit_prioritised_package.m_state.GetRejectReason()); - const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, - GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); - BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); - auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); - auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } // Package feerate is calculated without topology in mind; it's just aggregating fees and sizes. @@ -820,31 +831,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_rich_parent, /*test_accept=*/false); + if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_rich_parent.value()); + } else { + // The child would have been validated on its own and failed. + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); + + auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); + auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); + BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, + strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); + BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); + BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); + } expected_pool_size += 1; - BOOST_CHECK_MESSAGE(submit_rich_parent.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - - // The child would have been validated on its own and failed. - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); - - auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); - auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); - BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, - strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); - BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent_rich->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_poor->GetHash()))); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index bf5a653090..e0404e33ed 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -4,11 +4,15 @@ #include <test/util/net.h> -#include <chainparams.h> -#include <node/eviction.h> #include <net.h> #include <net_processing.h> +#include <netaddress.h> #include <netmessagemaker.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <protocol.h> +#include <random.h> +#include <serialize.h> #include <span.h> #include <vector> @@ -98,6 +102,17 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) co return complete; } +CNode* ConnmanTestMsg::ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) +{ + CNode* node = ConnectNode(CAddress{}, pszDest, /*fCountFailure=*/false, conn_type, /*use_v2transport=*/true); + if (!node) return nullptr; + node->SetCommonVersion(PROTOCOL_VERSION); + peerman.InitializeNode(*node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node->fSuccessfullyConnected = true; + AddTestNode(*node); + return node; +} + std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) { std::vector<NodeEvictionCandidate> candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index 497292542b..59c4ddb4b1 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,16 +6,30 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat/compat.h> -#include <node/eviction.h> -#include <netaddress.h> #include <net.h> +#include <net_permissions.h> +#include <net_processing.h> +#include <netaddress.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <sync.h> #include <util/sock.h> +#include <algorithm> #include <array> #include <cassert> +#include <chrono> +#include <cstdint> #include <cstring> #include <memory> #include <string> +#include <unordered_map> +#include <vector> + +class FastRandomContext; + +template <typename C> +class Span; struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; @@ -25,6 +39,12 @@ struct ConnmanTestMsg : public CConnman { m_peer_connect_timeout = timeout; } + std::vector<CNode*> TestNodes() + { + LOCK(m_nodes_mutex); + return m_nodes; + } + void AddTestNode(CNode& node) { LOCK(m_nodes_mutex); @@ -56,6 +76,11 @@ struct ConnmanTestMsg : public CConnman { bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const; void FlushSendBuffer(CNode& node) const; + + bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); }; + + CNode* ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index e70c105c8a..50cad32f10 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -49,6 +49,7 @@ #include <txdb.h> #include <txmempool.h> #include <util/chaintype.h> +#include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> #include <util/thread.h> @@ -146,6 +147,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto BasicTestingSetup::~BasicTestingSetup() { + m_node.kernel.reset(); SetMockTime(0s); // Reset mocktime for following tests LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); @@ -204,8 +206,9 @@ ChainTestingSetup::~ChainTestingSetup() m_node.netgroupman.reset(); m_node.args = nullptr; m_node.mempool.reset(); - m_node.scheduler.reset(); + m_node.fee_estimator.reset(); m_node.chainman.reset(); + m_node.scheduler.reset(); } void ChainTestingSetup::LoadVerifyActivateChainstate() @@ -336,56 +339,106 @@ CBlock TestChain100Setup::CreateAndProcessBlock( return block; } - -CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, - int input_vout, - int input_height, - CKey input_signing_key, - CScript output_destination, - CAmount output_amount, - bool submit) +std::pair<CMutableTransaction, CAmount> TestChain100Setup::CreateValidTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + const std::optional<CFeeRate>& feerate, + const std::optional<uint32_t>& fee_output) { - // Transaction we will submit to the mempool CMutableTransaction mempool_txn; + mempool_txn.vin.reserve(inputs.size()); + mempool_txn.vout.reserve(outputs.size()); - // Create an input - COutPoint outpoint_to_spend(input_transaction->GetHash(), input_vout); - CTxIn input(outpoint_to_spend); - mempool_txn.vin.push_back(input); - - // Create an output - CTxOut output(output_amount, output_destination); - mempool_txn.vout.push_back(output); + for (const auto& outpoint : inputs) { + mempool_txn.vin.emplace_back(outpoint, CScript(), MAX_BIP125_RBF_SEQUENCE); + } + mempool_txn.vout = outputs; - // Sign the transaction // - Add the signing key to a keystore FillableSigningProvider keystore; - keystore.AddKey(input_signing_key); + for (const auto& input_signing_key : input_signing_keys) { + keystore.AddKey(input_signing_key); + } // - Populate a CoinsViewCache with the unspent output CCoinsView coins_view; CCoinsViewCache coins_cache(&coins_view); - AddCoins(coins_cache, *input_transaction.get(), input_height); - // - Use GetCoin to properly populate utxo_to_spend, - Coin utxo_to_spend; - assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); - // - Then add it to a map to pass in to SignTransaction + for (const auto& input_transaction : input_transactions) { + AddCoins(coins_cache, *input_transaction.get(), input_height); + } + // Build Outpoint to Coin map for SignTransaction std::map<COutPoint, Coin> input_coins; - input_coins.insert({outpoint_to_spend, utxo_to_spend}); + CAmount inputs_amount{0}; + for (const auto& outpoint_to_spend : inputs) { + // - Use GetCoin to properly populate utxo_to_spend, + Coin utxo_to_spend; + assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); + input_coins.insert({outpoint_to_spend, utxo_to_spend}); + inputs_amount += utxo_to_spend.out.nValue; + } // - Default signature hashing type int nHashType = SIGHASH_ALL; std::map<int, bilingual_str> input_errors; assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + CAmount current_fee = inputs_amount - std::accumulate(outputs.begin(), outputs.end(), CAmount(0), + [](const CAmount& acc, const CTxOut& out) { + return acc + out.nValue; + }); + // Deduct fees from fee_output to meet feerate if set + if (feerate.has_value()) { + assert(fee_output.has_value()); + assert(fee_output.value() < mempool_txn.vout.size()); + CAmount target_fee = feerate.value().GetFee(GetVirtualTransactionSize(CTransaction{mempool_txn})); + CAmount deduction = target_fee - current_fee; + if (deduction > 0) { + // Only deduct fee if there's anything to deduct. If the caller has put more fees than + // the target feerate, don't change the fee. + mempool_txn.vout[fee_output.value()].nValue -= deduction; + // Re-sign since an output has changed + input_errors.clear(); + assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + current_fee = target_fee; + } + } + return {mempool_txn, current_fee}; +} +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + bool submit) +{ + CMutableTransaction mempool_txn = CreateValidTransaction(input_transactions, inputs, input_height, input_signing_keys, outputs, std::nullopt, std::nullopt).first; // If submit=true, add transaction to the mempool. if (submit) { LOCK(cs_main); const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(mempool_txn)); assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } - return mempool_txn; } +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, + uint32_t input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount, + bool submit) +{ + COutPoint input{input_transaction->GetHash(), input_vout}; + CTxOut output{output_amount, output_destination}; + return CreateValidMempoolTransaction(/*input_transactions=*/{input_transaction}, + /*inputs=*/{input}, + /*input_height=*/input_height, + /*input_signing_keys=*/{input_signing_key}, + /*outputs=*/{output}, + /*submit=*/submit); +} + std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContext& det_rand, size_t num_transactions, bool submit) { std::vector<CTransactionRef> mempool_transactions; diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 7cd4fdb417..4e1a26f303 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -124,7 +124,44 @@ struct TestChain100Setup : public TestingSetup { void mineBlocks(int num_blocks); /** - * Create a transaction and submit to the mempool. + * Create a transaction, optionally setting the fee based on the feerate. + * Note: The feerate may not be met exactly depending on whether the signatures can have different sizes. + * + * @param input_transactions The transactions to spend + * @param inputs Outpoints with which to construct transaction vin. + * @param input_height The height of the block that included the input transactions. + * @param input_signing_keys The keys to spend the input transactions. + * @param outputs Transaction vout. + * @param feerate The feerate the transaction should pay. + * @param fee_output The index of the output to take the fee from. + * @return The transaction and the fee it pays + */ + std::pair<CMutableTransaction, CAmount> CreateValidTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + const std::optional<CFeeRate>& feerate, + const std::optional<uint32_t>& fee_output); + /** + * Create a transaction and, optionally, submit to the mempool. + * + * @param input_transactions The transactions to spend + * @param inputs Outpoints with which to construct transaction vin. + * @param input_height The height of the block that included the input transaction(s). + * @param input_signing_keys The keys to spend inputs. + * @param outputs Transaction vout. + * @param submit Whether or not to submit to mempool + */ + CMutableTransaction CreateValidMempoolTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + bool submit = true); + + /** + * Create a 1-in-1-out transaction and, optionally, submit to the mempool. * * @param input_transaction The transaction to spend * @param input_vout The vout to spend from the input_transaction @@ -135,7 +172,7 @@ struct TestChain100Setup : public TestingSetup { * @param submit Whether or not to submit to mempool */ CMutableTransaction CreateValidMempoolTransaction(CTransactionRef input_transaction, - int input_vout, + uint32_t input_vout, int input_height, CKey input_signing_key, CScript output_destination, diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index c945f35d79..6d3bb01be8 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -11,6 +11,7 @@ #include <util/check.h> #include <util/time.h> #include <util/translation.h> +#include <validation.h> using node::NodeContext; @@ -36,3 +37,83 @@ CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const { return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, m_sequence, spendsCoinbase, sigOpCost, lp}; } + +std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, + const PackageMempoolAcceptResult& result, + bool expect_valid, + const CTxMemPool* mempool) +{ + if (expect_valid) { + if (result.m_state.IsInvalid()) { + return strprintf("Package validation unexpectedly failed: %s", result.m_state.ToString()); + } + } else { + if (result.m_state.IsValid()) { + return strprintf("Package validation unexpectedly succeeded. %s", result.m_state.ToString()); + } + } + if (result.m_state.GetResult() != PackageValidationResult::PCKG_POLICY && txns.size() != result.m_tx_results.size()) { + return strprintf("txns size %u does not match tx results size %u", txns.size(), result.m_tx_results.size()); + } + for (const auto& tx : txns) { + const auto& wtxid = tx->GetWitnessHash(); + if (result.m_tx_results.count(wtxid) == 0) { + return strprintf("result not found for tx %s", wtxid.ToString()); + } + + const auto& atmp_result = result.m_tx_results.at(wtxid); + const bool valid{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID}; + if (expect_valid && atmp_result.m_state.IsInvalid()) { + return strprintf("tx %s unexpectedly failed: %s", wtxid.ToString(), atmp_result.m_state.ToString()); + } + + //m_replaced_transactions should exist iff the result was VALID + if (atmp_result.m_replaced_transactions.has_value() != valid) { + return strprintf("tx %s result should %shave m_replaced_transactions", + wtxid.ToString(), valid ? "" : "not "); + } + + // m_vsize and m_base_fees should exist iff the result was VALID or MEMPOOL_ENTRY + const bool mempool_entry{atmp_result.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY}; + if (atmp_result.m_base_fees.has_value() != (valid || mempool_entry)) { + return strprintf("tx %s result should %shave m_base_fees", wtxid.ToString(), valid || mempool_entry ? "" : "not "); + } + if (atmp_result.m_vsize.has_value() != (valid || mempool_entry)) { + return strprintf("tx %s result should %shave m_vsize", wtxid.ToString(), valid || mempool_entry ? "" : "not "); + } + + // m_other_wtxid should exist iff the result was DIFFERENT_WITNESS + const bool diff_witness{atmp_result.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS}; + if (atmp_result.m_other_wtxid.has_value() != diff_witness) { + return strprintf("tx %s result should %shave m_other_wtxid", wtxid.ToString(), diff_witness ? "" : "not "); + } + + // m_effective_feerate and m_wtxids_fee_calculations should exist iff the result was valid + // or if the failure was TX_RECONSIDERABLE + const bool valid_or_reconsiderable{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID || + atmp_result.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + if (atmp_result.m_effective_feerate.has_value() != valid_or_reconsiderable) { + return strprintf("tx %s result should %shave m_effective_feerate", + wtxid.ToString(), valid ? "" : "not "); + } + if (atmp_result.m_wtxids_fee_calculations.has_value() != valid_or_reconsiderable) { + return strprintf("tx %s result should %shave m_effective_feerate", + wtxid.ToString(), valid ? "" : "not "); + } + + if (mempool) { + // The tx by txid should be in the mempool iff the result was not INVALID. + const bool txid_in_mempool{atmp_result.m_result_type != MempoolAcceptResult::ResultType::INVALID}; + if (mempool->exists(GenTxid::Txid(tx->GetHash())) != txid_in_mempool) { + return strprintf("tx %s should %sbe in mempool", wtxid.ToString(), txid_in_mempool ? "" : "not "); + } + // Additionally, if the result was DIFFERENT_WITNESS, we shouldn't be able to find the tx in mempool by wtxid. + if (tx->HasWitness() && atmp_result.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + if (mempool->exists(GenTxid::Wtxid(wtxid))) { + return strprintf("wtxid %s should not be in mempool", wtxid.ToString()); + } + } + } + } + return std::nullopt; +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h index 4b0daf0d42..a866d1ce74 100644 --- a/src/test/util/txmempool.h +++ b/src/test/util/txmempool.h @@ -5,12 +5,14 @@ #ifndef BITCOIN_TEST_UTIL_TXMEMPOOL_H #define BITCOIN_TEST_UTIL_TXMEMPOOL_H +#include <policy/packages.h> #include <txmempool.h> #include <util/time.h> namespace node { struct NodeContext; } +struct PackageMempoolAcceptResult; CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); @@ -36,4 +38,12 @@ struct TestMemPoolEntryHelper { TestMemPoolEntryHelper& SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } }; +/** Check expected properties for every PackageMempoolAcceptResult, regardless of value. Returns + * a string if an error occurs with error populated, nullopt otherwise. If mempool is provided, + * checks that the expected transactions are in mempool (this should be set to nullptr for a test_accept). +*/ +std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, + const PackageMempoolAcceptResult& result, + bool expect_valid, + const CTxMemPool* mempool); #endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 64cb5522eb..35e5c6a037 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -283,8 +283,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Check that all txs are in the pool { - LOCK(m_node.mempool->cs); - BOOST_CHECK_EQUAL(m_node.mempool->mapTx.size(), txs.size()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), txs.size()); } // Run a thread that simulates an RPC caller that is polling while @@ -295,7 +294,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // not some intermediate amount. while (true) { LOCK(m_node.mempool->cs); - if (m_node.mempool->mapTx.size() == 0) { + if (m_node.mempool->size() == 0) { // We are done with the reorg break; } @@ -304,7 +303,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // be atomic. So the caller assumes that the returned mempool // is consistent. That is, it has all txs that were there // before the reorg. - assert(m_node.mempool->mapTx.size() == txs.size()); + assert(m_node.mempool->size() == txs.size()); continue; } LOCK(cs_main); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 227d7d4633..e2541a74fd 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -579,7 +579,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) // it will initialize instead of attempting to complete validation. // // Note that this is not a realistic use of DisconnectTip(). - DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_BYTES}; BlockValidationState unused_state; { LOCK2(::cs_main, bg_chainstate.MempoolMutex()); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 461662ad93..efcb77f47c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -480,8 +480,8 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces minerPolicyEstimator->processTransaction(entry, validFeeEstimate); } - vTxHashes.emplace_back(tx.GetWitnessHash(), newit); - newit->vTxHashesIdx = vTxHashes.size() - 1; + txns_randomized.emplace_back(newit->GetSharedTx()); + newit->idx_randomized = txns_randomized.size() - 1; TRACE3(mempool, added, entry.GetTx().GetHash().data(), @@ -517,14 +517,16 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); - if (vTxHashes.size() > 1) { - vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); - vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; - vTxHashes.pop_back(); - if (vTxHashes.size() * 2 < vTxHashes.capacity()) - vTxHashes.shrink_to_fit(); + if (txns_randomized.size() > 1) { + // Update idx_randomized of the to-be-moved entry. + Assert(GetEntry(txns_randomized.back()->GetHash()))->idx_randomized = it->idx_randomized; + // Remove entry from txns_randomized by replacing it with the back and deleting the back. + txns_randomized[it->idx_randomized] = std::move(txns_randomized.back()); + txns_randomized.pop_back(); + if (txns_randomized.size() * 2 < txns_randomized.capacity()) + txns_randomized.shrink_to_fit(); } else - vTxHashes.clear(); + txns_randomized.clear(); totalTxSize -= it->GetTxSize(); m_total_fee -= it->GetFee(); @@ -836,6 +838,18 @@ static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), it->GetFee(), it->GetTxSize(), it->GetModifiedFee() - it->GetFee()}; } +std::vector<CTxMemPoolEntryRef> CTxMemPool::entryAll() const +{ + AssertLockHeld(cs); + + std::vector<CTxMemPoolEntryRef> ret; + ret.reserve(mapTx.size()); + for (const auto& it : GetSortedDepthAndScore()) { + ret.emplace_back(*it); + } + return ret; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -850,6 +864,13 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const return ret; } +const CTxMemPoolEntry* CTxMemPool::GetEntry(const Txid& txid) const +{ + AssertLockHeld(cs); + const auto i = mapTx.find(txid); + return i == mapTx.end() ? nullptr : &(*i); +} + CTransactionRef CTxMemPool::get(const uint256& hash) const { LOCK(cs); @@ -1033,7 +1054,7 @@ void CCoinsViewMemPool::Reset() size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(txns_randomized) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { diff --git a/src/txmempool.h b/src/txmempool.h index cbeabb31fa..3b0b8cf519 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -392,7 +392,7 @@ public: indexed_transaction_set mapTx GUARDED_BY(cs); using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; - std::vector<std::pair<uint256, txiter>> vTxHashes GUARDED_BY(cs); //!< All tx witness hashes/entries in mapTx, in random order + std::vector<CTransactionRef> txns_randomized GUARDED_BY(cs); //!< All transactions in mapTx, in random order typedef std::set<txiter, CompareIteratorByHash> setEntries; @@ -684,6 +684,8 @@ public: return (mapTx.count(gtxid.GetHash()) != 0); } + const CTxMemPoolEntry* GetEntry(const Txid& txid) const LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(cs); + CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) { @@ -695,6 +697,7 @@ public: /** Returns info for a transaction if its entry_sequence < last_sequence */ TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<CTxMemPoolEntryRef> entryAll() const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/util/sock.cpp b/src/util/sock.cpp index d16dc56aa3..e896b87160 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -242,7 +242,7 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per #endif /* USE_POLL */ } -void Sock::SendComplete(const std::string& data, +void Sock::SendComplete(Span<const unsigned char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const { @@ -283,6 +283,13 @@ void Sock::SendComplete(const std::string& data, } } +void Sock::SendComplete(Span<const char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + SendComplete(MakeUCharSpan(data), timeout, interrupt); +} + std::string Sock::RecvUntilTerminator(uint8_t terminator, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt, diff --git a/src/util/sock.h b/src/util/sock.h index d78e01929b..65e7ffc165 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -228,7 +228,14 @@ public: * @throws std::runtime_error if the operation cannot be completed. In this case only some of * the data will be written to the socket. */ - virtual void SendComplete(const std::string& data, + virtual void SendComplete(Span<const unsigned char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Convenience method, equivalent to `SendComplete(MakeUCharSpan(data), timeout, interrupt)`. + */ + virtual void SendComplete(Span<const char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index d792562735..439678c24a 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -260,7 +260,6 @@ bool TimingResistantEqual(const T& a, const T& b) } /** Parse number as fixed point according to JSON number syntax. - * See https://json.org/number.gif * @returns true on success, false on error. * @note The result must be in the range (-10^18,10^18), otherwise an overflow error will trigger. */ diff --git a/src/validation.cpp b/src/validation.cpp index c72188d581..34103d18bc 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,10 +5,6 @@ #include <validation.h> -#include <kernel/chain.h> -#include <kernel/coinstats.h> -#include <kernel/mempool_persist.h> - #include <arith_uint256.h> #include <chain.h> #include <checkqueue.h> @@ -22,7 +18,9 @@ #include <cuckoocache.h> #include <flatfile.h> #include <hash.h> +#include <kernel/chain.h> #include <kernel/chainparams.h> +#include <kernel/coinstats.h> #include <kernel/disconnected_transactions.h> #include <kernel/mempool_entry.h> #include <kernel/messagestartchars.h> @@ -355,7 +353,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( const std::optional<LockPoints> new_lock_points{CalculateLockPointsAtTip(m_chain.Tip(), view_mempool, tx)}; if (new_lock_points.has_value() && CheckSequenceLocksAtTip(m_chain.Tip(), *new_lock_points)) { // Now update the mempool entry lockpoints as well. - m_mempool->mapTx.modify(it, [&new_lock_points](CTxMemPoolEntry& e) { e.UpdateLockPoints(*new_lock_points); }); + it->UpdateLockPoints(*new_lock_points); } else { return true; } @@ -364,9 +362,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( // If the transaction spends any coinbase outputs, it must be mature. if (it->GetSpendsCoinbase()) { for (const CTxIn& txin : tx.vin) { - auto it2 = m_mempool->mapTx.find(txin.prevout.hash); - if (it2 != m_mempool->mapTx.end()) - continue; + if (m_mempool->exists(GenTxid::Txid(txin.prevout.hash))) continue; const Coin& coin{CoinsTip().AccessCoin(txin.prevout)}; assert(!coin.IsSpent()); const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; @@ -548,6 +544,9 @@ public: } }; + /** Clean up all non-chainstate coins from m_view and m_viewmempool. */ + void CleanupTemporaryCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Single transaction acceptance MempoolAcceptResult AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -669,11 +668,11 @@ private: AssertLockHeld(m_pool.cs); CAmount mempoolRejectFee = m_pool.GetMinFee().GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < m_pool.m_min_relay_feerate.GetFee(package_size)) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "min relay fee not met", strprintf("%d < %d", package_fee, m_pool.m_min_relay_feerate.GetFee(package_size))); } return true; @@ -866,6 +865,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. if (!bypass_limits && ws.m_modified_fees < m_pool.m_min_relay_feerate.GetFee(ws.m_vsize)) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", ws.m_modified_fees, m_pool.m_min_relay_feerate.GetFee(ws.m_vsize))); } @@ -980,6 +981,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } @@ -1001,6 +1005,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) } if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_incremental_relay_feerate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; @@ -1138,7 +1145,8 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) if (!args.m_package_submission && !bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); if (!m_pool.exists(GenTxid::Txid(hash))) - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); + // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); } return true; } @@ -1200,7 +1208,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& } } - std::vector<uint256> all_package_wtxids; + std::vector<Wtxid> all_package_wtxids; all_package_wtxids.reserve(workspaces.size()); std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); @@ -1210,7 +1218,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>({ws.m_ptx->GetWitnessHash()}); + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); @@ -1225,8 +1233,15 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) Workspace ws(ptx); + const std::vector<Wtxid> single_wtxid{ws.m_ptx->GetWitnessHash()}; - if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!PreChecks(args, ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); @@ -1237,14 +1252,18 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); const CFeeRate effective_feerate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; - const std::vector<uint256> single_wtxid{ws.m_ptx->GetWitnessHash()}; // Tx was accepted, but not added if (args.m_test_accept) { return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } - if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!Finalize(args, ws)) { + // The only possible failure reason is fee-related (mempool full). + // Failed for fee reasons. Provide the effective feerate and which txns were included. + Assume(ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE); + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + } GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); @@ -1258,7 +1277,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // These context-free package limits can be done before taking the mempool lock. PackageValidationState package_state; - if (!CheckPackage(txns, package_state)) return PackageMempoolAcceptResult(package_state, {}); + if (!IsWellFormedPackage(txns, package_state, /*require_sorted=*/true)) return PackageMempoolAcceptResult(package_state, {}); std::vector<Workspace> workspaces{}; workspaces.reserve(txns.size()); @@ -1298,11 +1317,16 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + std::vector<Wtxid> all_package_wtxids; + all_package_wtxids.reserve(workspaces.size()); + std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), + [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); TxValidationState placeholder_state; if (args.m_package_feerates && !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-fee-too-low"); - return PackageMempoolAcceptResult(package_state, {}); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + return PackageMempoolAcceptResult(package_state, {{workspaces.back().m_ptx->GetWitnessHash(), + MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_total_modified_fees, m_total_vsize), all_package_wtxids)}}); } // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, @@ -1313,10 +1337,6 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } - std::vector<uint256> all_package_wtxids; - all_package_wtxids.reserve(workspaces.size()); - std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), - [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); for (Workspace& ws : workspaces) { ws.m_package_feerate = package_feerate; if (!PolicyScriptChecks(args, ws)) { @@ -1329,7 +1349,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>{ws.m_ptx->GetWitnessHash()}; + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, @@ -1347,26 +1367,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } -PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTransactionRef>& subpackage, ATMPArgs& args) +void MemPoolAccept::CleanupTemporaryCoins() { - AssertLockHeld(::cs_main); - AssertLockHeld(m_pool.cs); - auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_pool.cs) { - if (subpackage.size() > 1) { - return AcceptMultipleTransactions(subpackage, args); - } - const auto& tx = subpackage.front(); - ATMPArgs single_args = ATMPArgs::SingleInPackageAccept(args); - const auto single_res = AcceptSingleTransaction(tx, single_args); - PackageValidationState package_state_wrapped; - if (single_res.m_result_type != MempoolAcceptResult::ResultType::VALID) { - package_state_wrapped.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); - } - return PackageMempoolAcceptResult(package_state_wrapped, {{tx->GetWitnessHash(), single_res}}); - }(); - // Clean up m_view and m_viewmempool so that other subpackage evaluations don't have access to - // coins they shouldn't. Keep some coins in order to minimize re-fetching coins from the UTXO set. - // // There are 3 kinds of coins in m_view: // (1) Temporary coins from the transactions in subpackage, constructed by m_viewmempool. // (2) Mempool coins from transactions in the mempool, constructed by m_viewmempool. @@ -1392,6 +1394,30 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTr } // This deletes the temporary and mempool coins. m_viewmempool.Reset(); +} + +PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTransactionRef>& subpackage, ATMPArgs& args) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(m_pool.cs); + auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_pool.cs) { + if (subpackage.size() > 1) { + return AcceptMultipleTransactions(subpackage, args); + } + const auto& tx = subpackage.front(); + ATMPArgs single_args = ATMPArgs::SingleInPackageAccept(args); + const auto single_res = AcceptSingleTransaction(tx, single_args); + PackageValidationState package_state_wrapped; + if (single_res.m_result_type != MempoolAcceptResult::ResultType::VALID) { + package_state_wrapped.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + } + return PackageMempoolAcceptResult(package_state_wrapped, {{tx->GetWitnessHash(), single_res}}); + }(); + + // Clean up m_view and m_viewmempool so that other subpackage evaluations don't have access to + // coins they shouldn't. Keep some coins in order to minimize re-fetching coins from the UTXO set. + CleanupTemporaryCoins(); + return result; } @@ -1405,7 +1431,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions and thus won't return any MempoolAcceptResults, just a package-wide error. // Context-free package checks. - if (!CheckPackage(package, package_state_quit_early)) return PackageMempoolAcceptResult(package_state_quit_early, {}); + if (!IsWellFormedPackage(package, package_state_quit_early, /*require_sorted=*/true)) { + return PackageMempoolAcceptResult(package_state_quit_early, {}); + } // All transactions in the package must be a parent of the last transaction. This is just an // opportunity for us to fail fast on a context-free check without taking the mempool lock. @@ -1476,9 +1504,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with // the new transactions. This ensures we don't double-count transaction counts and sizes when // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(iter.value()->GetTxSize(), iter.value()->GetFee())); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(entry.GetTxSize(), entry.GetFee())); } else if (m_pool.exists(GenTxid::Txid(txid))) { // Transaction with the same non-witness data but different witness (same txid, // different wtxid) already exists in the mempool. @@ -1487,10 +1514,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transaction for the mempool one. Note that we are ignoring the validity of the // package transaction passed in. // TODO: allow witness replacement in packages. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; // Provide the wtxid of the mempool tx so that the caller can look it up in the mempool. - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(entry.GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. // Try submitting the transaction on its own. @@ -1501,7 +1527,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // in package validation, because its fees should only be "used" once. assert(m_pool.exists(GenTxid::Wtxid(wtxid))); results_final.emplace(wtxid, single_res); - } else if (single_res.m_state.GetResult() != TxValidationResult::TX_MEMPOOL_POLICY && + } else if (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { // Package validation policy only differs from individual policy in its evaluation // of feerate. For example, if a transaction fails here due to violation of a @@ -3064,7 +3090,7 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; while (m_chain.Tip() && m_chain.Tip() != pindexFork) { if (!DisconnectTip(state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, @@ -3422,7 +3448,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // ActivateBestChain considers blocks already in m_chain // unconditionally valid already, so force disconnect away from it. - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; bool ret = DisconnectTip(state, &disconnectpool); // DisconnectTip will add transactions to disconnectpool. // Adjust the mempool to be consistent with the new tip, adding diff --git a/src/validation.h b/src/validation.h index 7ce60da634..e669ec46d5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -114,7 +114,27 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); /** -* Validation result for a single transaction mempool acceptance. +* Validation result for a transaction evaluated by MemPoolAccept (single or package). +* Here are the expected fields and properties of a result depending on its ResultType, applicable to +* results returned from package evaluation: +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| Field or property | VALID | INVALID | MEMPOOL_ENTRY | DIFFERENT_WITNESS | +*| | |--------------------------------------| | | +*| | | TX_RECONSIDERABLE | Other | | | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| txid in mempool? | yes | no | no* | yes | yes | +*| wtxid in mempool? | yes | no | no* | yes | no | +*| m_state | yes, IsValid() | yes, IsInvalid() | yes, IsInvalid() | yes, IsValid() | yes, IsValid() | +*| m_replaced_transactions | yes | no | no | no | no | +*| m_vsize | yes | no | no | yes | no | +*| m_base_fees | yes | no | no | yes | no | +*| m_effective_feerate | yes | yes | no | no | no | +*| m_wtxids_fee_calculations | yes | yes | no | no | no | +*| m_other_wtxid | no | no | no | no | yes | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +* (*) Individual transaction acceptance doesn't return MEMPOOL_ENTRY and DIFFERENT_WITNESS. It returns +* INVALID, with the errors txn-already-in-mempool and txn-same-nonwitness-data-in-mempool +* respectively. In those cases, the txid or wtxid may be in the mempool for a TX_CONFLICT. */ struct MempoolAcceptResult { /** Used to indicate the results of mempool validation. */ @@ -130,7 +150,6 @@ struct MempoolAcceptResult { /** Contains information about why the transaction failed. */ const TxValidationState m_state; - // The following fields are only present when m_result_type = ResultType::VALID or MEMPOOL_ENTRY /** Mempool transactions replaced by the tx. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ @@ -141,7 +160,6 @@ struct MempoolAcceptResult { * using prioritisetransaction (i.e. modified fees). If this transaction was submitted as a * package, this is the package feerate, which may also include its descendants and/or * ancestors (see m_wtxids_fee_calculations below). - * Only present when m_result_type = ResultType::VALID. */ const std::optional<CFeeRate> m_effective_feerate; /** Contains the wtxids of the transactions used for fee-related checks. Includes this @@ -149,9 +167,8 @@ struct MempoolAcceptResult { * package. This is not necessarily equivalent to the list of transactions passed to * ProcessNewPackage(). * Only present when m_result_type = ResultType::VALID. */ - const std::optional<std::vector<uint256>> m_wtxids_fee_calculations; + const std::optional<std::vector<Wtxid>> m_wtxids_fee_calculations; - // The following field is only present when m_result_type = ResultType::DIFFERENT_WITNESS /** The wtxid of the transaction in the mempool which has the same txid but different witness. */ const std::optional<uint256> m_other_wtxid; @@ -159,11 +176,17 @@ struct MempoolAcceptResult { return MempoolAcceptResult(state); } + static MempoolAcceptResult FeeFailure(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) { + return MempoolAcceptResult(state, effective_feerate, wtxids_fee_calculations); + } + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) { + const std::vector<Wtxid>& wtxids_fee_calculations) { return MempoolAcceptResult(std::move(replaced_txns), vsize, fees, effective_feerate, wtxids_fee_calculations); } @@ -189,7 +212,7 @@ private: int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) + const std::vector<Wtxid>& wtxids_fee_calculations) : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, @@ -197,6 +220,15 @@ private: m_effective_feerate(effective_feerate), m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for fee-related failure case */ + explicit MempoolAcceptResult(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) + : m_result_type(ResultType::INVALID), + m_state(state), + m_effective_feerate(effective_feerate), + m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for already-in-mempool case. It wouldn't replace any transactions. */ explicit MempoolAcceptResult(int64_t vsize, CAmount fees) : m_result_type(ResultType::MEMPOOL_ENTRY), m_vsize{vsize}, m_base_fees(fees) {} diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 0d0a8650ac..088343458c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -95,8 +95,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-walletcrosschain", strprintf("Allow reusing wallet files across chains (default: %u)", DEFAULT_WALLETCROSSCHAIN), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - - argsman.AddHiddenArgs({"-zapwallettxes"}); } bool WalletInit::ParameterInteraction() const @@ -118,10 +116,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.IsArgSet("-zapwallettxes")) { - return InitError(Untranslated("-zapwallettxes has been removed. If you are attempting to remove a stuck transaction from your wallet, please use abandontransaction instead.")); - } - return true; } diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index bc3327cdb2..d2b2801aa8 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -2601,7 +2601,7 @@ std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDe uint256 DescriptorScriptPubKeyMan::GetID() const { LOCK(cs_desc_man); - return DescriptorID(*m_wallet_descriptor.descriptor); + return m_wallet_descriptor.id; } void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) @@ -2655,7 +2655,7 @@ bool DescriptorScriptPubKeyMan::AddCryptedKey(const CKeyID& key_id, const CPubKe bool DescriptorScriptPubKeyMan::HasWalletDescriptor(const WalletDescriptor& desc) const { LOCK(cs_desc_man); - return m_wallet_descriptor.descriptor != nullptr && desc.descriptor != nullptr && m_wallet_descriptor.descriptor->ToString() == desc.descriptor->ToString(); + return !m_wallet_descriptor.id.IsNull() && !desc.id.IsNull() && m_wallet_descriptor.id == desc.id; } void DescriptorScriptPubKeyMan::WriteDescriptor() diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 4f78fe7520..6777257e53 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -4,6 +4,10 @@ #include <wallet/transaction.h> +#include <interfaces/chain.h> + +using interfaces::FoundBlock; + namespace wallet { bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { @@ -25,6 +29,27 @@ int64_t CWalletTx::GetTxTime() const return n ? n : nTimeReceived; } +void CWalletTx::updateState(interfaces::Chain& chain) +{ + bool active; + auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. + if (!chain.findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { + state = TxStateInactive{}; + } + }; + if (auto* conf = state<TxStateConfirmed>()) { + lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); + } else if (auto* conf = state<TxStateConflicted>()) { + lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); + } +} + void CWalletTx::CopyFrom(const CWalletTx& _tx) { *this = _tx; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index db858fa5ba..0c28628915 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -22,6 +22,10 @@ #include <variant> #include <vector> +namespace interfaces { +class Chain; +} // namespace interfaces + namespace wallet { //! State of transaction confirmed in a block. struct TxStateConfirmed { @@ -326,6 +330,10 @@ public: template<typename T> const T* state() const { return std::get_if<T>(&m_state); } template<typename T> T* state() { return std::get_if<T>(&m_state); } + //! Update transaction state when attaching to a chain, filling in heights + //! of conflicted and confirmed blocks + void updateState(interfaces::Chain& chain); + bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } bool isConflicted() const { return state<TxStateConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 162d7f9ec7..ecf18fbe78 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1184,23 +1184,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx // If wallet doesn't have a chain (e.g when using bitcoin-wallet tool), // don't bother to update txn. if (HaveChain()) { - bool active; - auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { - // If tx block (or conflicting block) was reorged out of chain - // while the wallet was shutdown, change tx status to UNCONFIRMED - // and reset block height, hash, and index. ABANDONED tx don't have - // associated blocks and don't need to be updated. The case where a - // transaction was reorged out while online and then reconfirmed - // while offline is covered by the rescan logic. - if (!chain().findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { - state = TxStateInactive{}; - } - }; - if (auto* conf = wtx.state<TxStateConfirmed>()) { - lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, wtx.m_state); - } else if (auto* conf = wtx.state<TxStateConflicted>()) { - lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, wtx.m_state); - } + wtx.updateState(chain()); } if (/* insertion took place */ ins.second) { wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -3319,8 +3303,10 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { AssertLockHeld(cs_wallet); if (auto* conf = wtx.state<TxStateConfirmed>()) { + assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; } else if (auto* conf = wtx.state<TxStateConflicted>()) { + assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { return 0; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 9333493a6e..0832887159 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -503,6 +503,13 @@ public: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain + * + * Preconditions: it is only valid to call this function when the wallet is + * online and the block index is loaded. So this cannot be called by + * bitcoin-wallet tool code or by wallet migration code. If this is called + * without the wallet being online, it won't be able able to determine the + * the height of the last block processed, or the heights of blocks + * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index c5975144c1..7ad3ffe9e4 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -85,6 +85,7 @@ class WalletDescriptor { public: std::shared_ptr<Descriptor> descriptor; + uint256 id; // Descriptor ID (calculated once at descriptor initialization/deserialization) uint64_t creation_time = 0; int32_t range_start = 0; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. int32_t range_end = 0; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() @@ -99,6 +100,7 @@ public: if (!descriptor) { throw std::ios_base::failure("Invalid descriptor: " + error); } + id = DescriptorID(*descriptor); } SERIALIZE_METHODS(WalletDescriptor, obj) @@ -110,7 +112,7 @@ public: } WalletDescriptor() {} - WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} + WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), id(DescriptorID(*descriptor)), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) { } }; } // namespace wallet diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index ab2e6c4d0b..d1232c5133 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -118,8 +118,14 @@ class AssumeutxoTest(BitcoinTestFramework): chainstate_snapshot_path.mkdir() with open(chainstate_snapshot_path / "base_blockhash", 'wb') as f: f.write(b'z' * 32) - expected_error = f"Error: A fatal internal error occurred, see debug.log for details" - self.nodes[0].assert_start_raises_init_error(expected_msg=expected_error) + + def expected_error(log_msg="", error_msg=""): + with self.nodes[0].assert_debug_log([log_msg]): + self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) + + expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details" + error_details = f"Assumeutxo data not found for the given blockhash" + expected_error(log_msg=error_details, error_msg=expected_error_msg) # resurrect node again rmtree(chainstate_snapshot_path) diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 94f5116f9b..142d75a851 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2021-2022 The Bitcoin Core developers +# Copyright (c) 2021-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Stress tests related to node initialization.""" @@ -133,15 +133,12 @@ class InitStressTest(BitcoinTestFramework): for target_file in target_files: self.log.info(f"Perturbing file to ensure failure {target_file}") - with open(target_file, "rb") as tf_read: - contents = tf_read.read() - tweaked_contents = bytearray(contents) + with open(target_file, "r+b") as tf: # Since the genesis block is not checked by -checkblocks, the # perturbation window must be chosen such that a higher block # in blk*.dat is affected. - tweaked_contents[150:350] = b'1' * 200 - with open(target_file, "wb") as tf_write: - tf_write.write(bytes(tweaked_contents)) + tf.seek(150) + tf.write(b"1" * 200) start_expecting_error(err_fragment) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 1d8c87feb5..e85541d0ec 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -104,7 +104,7 @@ from test_framework.key import ( tweak_add_privkey, ECKey, ) -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 from test_framework.address import ( hash160, program_to_witness, diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index ce2a5ab8ac..be154b411f 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -11,7 +11,7 @@ from test_framework.messages import ( COutPoint, from_hex, ) -from test_framework.muhash import MuHash3072 +from test_framework.crypto.muhash import MuHash3072 from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.wallet import MiniWallet diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 63cd10896d..2adcaf178c 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -270,15 +270,16 @@ class AddrTest(BitcoinTestFramework): full_outbound_peer.sync_with_ping() assert full_outbound_peer.getaddr_received() - self.log.info('Check that we do not send a getaddr message upon connecting to a block-relay-only peer') + self.log.info('Check that we do not send a getaddr message to a block-relay-only or inbound peer') block_relay_peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=1, connection_type="block-relay-only") block_relay_peer.sync_with_ping() assert_equal(block_relay_peer.getaddr_received(), False) - self.log.info('Check that we answer getaddr messages only from inbound peers') inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver(send_getaddr=False)) inbound_peer.sync_with_ping() + assert_equal(inbound_peer.getaddr_received(), False) + self.log.info('Check that we answer getaddr messages only from inbound peers') # Add some addresses to addrman for i in range(1000): first_octet = i >> 8 diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 2fb88b828f..4916d36ab7 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -216,7 +216,7 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', - '9.9.9.9:8333 mapped', + '9.9.9.9:8333', 'Added 1 addresses', ], bytes.fromhex( diff --git a/test/functional/p2p_v2_transport.py b/test/functional/p2p_v2_transport.py index 5ad2194b84..1a3b4a6d0a 100755 --- a/test/functional/p2p_v2_transport.py +++ b/test/functional/p2p_v2_transport.py @@ -48,7 +48,7 @@ class V2TransportTest(BitcoinTestFramework): assert_equal(self.nodes[1].getblockcount(), 5) # verify there is a v2 connection between node 0 and 1 node_0_info = self.nodes[0].getpeerinfo() - node_1_info = self.nodes[0].getpeerinfo() + node_1_info = self.nodes[1].getpeerinfo() assert_equal(len(node_0_info), 1) assert_equal(len(node_1_info), 1) assert_equal(node_0_info[0]["transport_protocol_type"], "v2") diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 53163720bb..8eb9f3aeb1 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -58,6 +58,7 @@ TIME_RANGE_STEP = 600 # ten-minute steps TIME_RANGE_MTP = TIME_GENESIS_BLOCK + (HEIGHT - 6) * TIME_RANGE_STEP TIME_RANGE_TIP = TIME_GENESIS_BLOCK + (HEIGHT - 1) * TIME_RANGE_STEP TIME_RANGE_END = TIME_GENESIS_BLOCK + HEIGHT * TIME_RANGE_STEP +DIFFICULTY_ADJUSTMENT_INTERVAL = 2016 class BlockchainTest(BitcoinTestFramework): @@ -451,6 +452,15 @@ class BlockchainTest(BitcoinTestFramework): # This should be 2 hashes every 10 minutes or 1/300 assert abs(hashes_per_second * 300 - 1) < 0.0001 + # Test setting the first param of getnetworkhashps to negative value returns the average network + # hashes per second from the last difficulty change. + current_block_height = self.nodes[0].getmininginfo()['blocks'] + blocks_since_last_diff_change = current_block_height % DIFFICULTY_ADJUSTMENT_INTERVAL + 1 + expected_hashes_per_second_since_diff_change = self.nodes[0].getnetworkhashps(blocks_since_last_diff_change) + + assert_equal(self.nodes[0].getnetworkhashps(-1), expected_hashes_per_second_since_diff_change) + assert_equal(self.nodes[0].getnetworkhashps(-2), expected_hashes_per_second_since_diff_change) + def _test_stopatheight(self): self.log.info("Test stopping at height") assert_equal(self.nodes[0].getblockcount(), HEIGHT) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 50a022fc7e..773ab3b50e 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -215,8 +215,11 @@ class NetTest(BitcoinTestFramework): # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') + # try to add an equivalent ip + ip_port2 = "127.1:{}".format(p2p_port(2)) + assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port2, command='add') # check that the node has indeed been added - added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + added_nodes = self.nodes[0].getaddednodeinfo() assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that node cannot be added again diff --git a/test/functional/test_framework/blockfilter.py b/test/functional/test_framework/blockfilter.py index a30e37ea5b..3d6b38a23d 100644 --- a/test/functional/test_framework/blockfilter.py +++ b/test/functional/test_framework/blockfilter.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helper routines relevant for compact block filters (BIP158). """ -from .siphash import siphash +from .crypto.siphash import siphash def bip158_basic_element_hash(script_pub_key, N, block_hash): diff --git a/test/functional/test_framework/crypto/bip324_cipher.py b/test/functional/test_framework/crypto/bip324_cipher.py new file mode 100644 index 0000000000..56190647f2 --- /dev/null +++ b/test/functional/test_framework/crypto/bip324_cipher.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +from .chacha20 import chacha20_block, REKEY_INTERVAL +from .poly1305 import Poly1305 + + +def pad16(x): + if len(x) % 16 == 0: + return b'' + return b'\x00' * (16 - (len(x) % 16)) + + +def aead_chacha20_poly1305_encrypt(key, nonce, aad, plaintext): + """Encrypt a plaintext using ChaCha20Poly1305.""" + ret = bytearray() + msg_len = len(plaintext) + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(plaintext[j + 64 * i] ^ keystream[j]) + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ret + pad16(ret) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + ret += poly1305.tag(mac_data) + return bytes(ret) + + +def aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext): + """Decrypt a ChaCha20Poly1305 ciphertext.""" + if len(ciphertext) < 16: + return None + msg_len = len(ciphertext) - 16 + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ciphertext[:-16] + pad16(ciphertext[:-16]) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + if ciphertext[-16:] != poly1305.tag(mac_data): + return None + ret = bytearray() + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(ciphertext[j + 64 * i] ^ keystream[j]) + return bytes(ret) + + +class FSChaCha20Poly1305: + """Rekeying wrapper AEAD around ChaCha20Poly1305.""" + def __init__(self, initial_key): + self._key = initial_key + self._packet_counter = 0 + + def _crypt(self, aad, text, is_decrypt): + nonce = ((self._packet_counter % REKEY_INTERVAL).to_bytes(4, 'little') + + (self._packet_counter // REKEY_INTERVAL).to_bytes(8, 'little')) + if is_decrypt: + ret = aead_chacha20_poly1305_decrypt(self._key, nonce, aad, text) + else: + ret = aead_chacha20_poly1305_encrypt(self._key, nonce, aad, text) + if (self._packet_counter + 1) % REKEY_INTERVAL == 0: + rekey_nonce = b"\xFF\xFF\xFF\xFF" + nonce[4:] + self._key = aead_chacha20_poly1305_encrypt(self._key, rekey_nonce, b"", b"\x00" * 32)[:32] + self._packet_counter += 1 + return ret + + def decrypt(self, aad, ciphertext): + return self._crypt(aad, ciphertext, True) + + def encrypt(self, aad, plaintext): + return self._crypt(aad, plaintext, False) + + +# Test vectors from RFC8439 consisting of plaintext, aad, 32 byte key, 12 byte nonce and ciphertext +AEAD_TESTS = [ + # RFC 8439 Example from section 2.8.2 + ["4c616469657320616e642047656e746c656d656e206f662074686520636c6173" + "73206f66202739393a204966204920636f756c64206f6666657220796f75206f" + "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73" + "637265656e20776f756c642062652069742e", + "50515253c0c1c2c3c4c5c6c7", + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f", + [7, 0x4746454443424140], + "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6" + "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36" + "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc" + "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060" + "0691"], + # RFC 8439 Test vector A.5 + ["496e7465726e65742d4472616674732061726520647261667420646f63756d65" + "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d" + "6f6e74687320616e64206d617920626520757064617465642c207265706c6163" + "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65" + "6e747320617420616e792074696d652e20497420697320696e617070726f7072" + "6961746520746f2075736520496e7465726e65742d4472616674732061732072" + "65666572656e6365206d6174657269616c206f7220746f206369746520746865" + "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67" + "726573732e2fe2809d", + "f33388860000000000004e91", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + [0, 0x0807060504030201], + "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2" + "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf" + "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855" + "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4" + "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e" + "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a" + "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10" + "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29" + "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"], + # Test vectors exercising aad and plaintext which are multiples of 16 bytes. + ["8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951" + "a6b7ad3db580be0674c3f0b55f618e34", + "", + "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3", + [0x3432b75f, 0xb3585537eb7f4024], + "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a" + "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"], + ["", + "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3" + "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503", + "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021", + [0x1f90da88, 0x75dafa3ef84471a4], + "aaae5bb81e8407c94b2ae86ae0c7efbe"], +] + +FSAEAD_TESTS = [ + ["d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e" + "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf" + "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60" + "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c" + "711191b14d75a72147", + "786cb9b6ebf44288974cf0", + "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654", + 500, + "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e" + "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75" + "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4" + "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192" + "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"], + ["8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc" + "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234", + "", + "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce", + 60000, + "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4" + "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c" + "14b94829deb27f0b1923a2af704ae5d6"], +] + + +class TestFrameworkAEAD(unittest.TestCase): + def test_aead(self): + """ChaCha20Poly1305 AEAD test vectors.""" + for test_vector in AEAD_TESTS: + hex_plain, hex_aad, hex_key, hex_nonce, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + nonce = hex_nonce[0].to_bytes(4, 'little') + hex_nonce[1].to_bytes(8, 'little') + + ciphertext = aead_chacha20_poly1305_encrypt(key, nonce, aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + plaintext = aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext) + self.assertEqual(plain, plaintext) + + def test_fschacha20poly1305aead(self): + "FSChaCha20Poly1305 AEAD test vectors." + for test_vector in FSAEAD_TESTS: + hex_plain, hex_aad, hex_key, msg_idx, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + + enc_aead = FSChaCha20Poly1305(key) + dec_aead = FSChaCha20Poly1305(key) + + for _ in range(msg_idx): + enc_aead.encrypt(b"", b"") + ciphertext = enc_aead.encrypt(aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + + for _ in range(msg_idx): + dec_aead.decrypt(b"", bytes(16)) + plaintext = dec_aead.decrypt(aad, ciphertext) + self.assertEqual(plain, plaintext) diff --git a/test/functional/test_framework/crypto/chacha20.py b/test/functional/test_framework/crypto/chacha20.py new file mode 100644 index 0000000000..19b6698dfb --- /dev/null +++ b/test/functional/test_framework/crypto/chacha20.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 cipher and FSChaCha20 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +CHACHA20_INDICES = ( + (0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15), + (0, 5, 10, 15), (1, 6, 11, 12), (2, 7, 8, 13), (3, 4, 9, 14) +) + +CHACHA20_CONSTANTS = (0x61707865, 0x3320646e, 0x79622d32, 0x6b206574) +REKEY_INTERVAL = 224 # packets + + +def rotl32(v, bits): + """Rotate the 32-bit value v left by bits bits.""" + bits %= 32 # Make sure the term below does not throw an exception + return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) + + +def chacha20_doubleround(s): + """Apply a ChaCha20 double round to 16-element state array s. + See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 + """ + for a, b, c, d in CHACHA20_INDICES: + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 16) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 12) + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 8) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 7) + + +def chacha20_block(key, nonce, cnt): + """Compute the 64-byte output of the ChaCha20 block function. + Takes as input a 32-byte key, 12-byte nonce, and 32-bit integer counter. + """ + # Initial state. + init = [0] * 16 + init[:4] = CHACHA20_CONSTANTS[:4] + init[4:12] = [int.from_bytes(key[i:i+4], 'little') for i in range(0, 32, 4)] + init[12] = cnt + init[13:16] = [int.from_bytes(nonce[i:i+4], 'little') for i in range(0, 12, 4)] + # Perform 20 rounds. + state = list(init) + for _ in range(10): + chacha20_doubleround(state) + # Add initial values back into state. + for i in range(16): + state[i] = (state[i] + init[i]) & 0xffffffff + # Produce byte output + return b''.join(state[i].to_bytes(4, 'little') for i in range(16)) + +class FSChaCha20: + """Rekeying wrapper stream cipher around ChaCha20.""" + def __init__(self, initial_key, rekey_interval=REKEY_INTERVAL): + self._key = initial_key + self._rekey_interval = rekey_interval + self._block_counter = 0 + self._chunk_counter = 0 + self._keystream = b'' + + def _get_keystream_bytes(self, nbytes): + while len(self._keystream) < nbytes: + nonce = ((0).to_bytes(4, 'little') + (self._chunk_counter // self._rekey_interval).to_bytes(8, 'little')) + self._keystream += chacha20_block(self._key, nonce, self._block_counter) + self._block_counter += 1 + ret = self._keystream[:nbytes] + self._keystream = self._keystream[nbytes:] + return ret + + def crypt(self, chunk): + ks = self._get_keystream_bytes(len(chunk)) + ret = bytes([ks[i] ^ chunk[i] for i in range(len(chunk))]) + if ((self._chunk_counter + 1) % self._rekey_interval) == 0: + self._key = self._get_keystream_bytes(32) + self._block_counter = 0 + self._keystream = b'' + self._chunk_counter += 1 + return ret + + +# Test vectors from RFC7539/8439 consisting of 32 byte key, 12 byte nonce, block counter +# and 64 byte output after applying `chacha20_block` function +CHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0x09000000, 0x4a000000], 1, + "10f1e7e4d13b5915500fdd1fa32071c4c7d1f4c733c068030422aa9ac3d46c4e" + "d2826446079faa0914c2d705d98b02a2b5129cd1de164eb9cbd083e8a2503c4e"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7" + "da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 1, + "9f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32ee7aed" + "29b721769ce64e43d57133b074d839d531ed1f28510afb45ace10a1f4b794d6f"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 1, + "3aeb5224ecf849929b9d828db1ced4dd832025e8018b8160b82284f3c949aa5a" + "8eca00bbb4a73bdad192b5c42f73f2fd4e273644c8b36125a64addeb006c13a0"], + ["00ff000000000000000000000000000000000000000000000000000000000000", [0, 0], 2, + "72d54dfbf12ec44b362692df94137f328fea8da73990265ec1bbbea1ae9af0ca" + "13b25aa26cb4a648cb9b9d1be65b2c0924a66c54d545ec1b7374f4872e99f096"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0x200000000000000], 0, + "c2c64d378cd536374ae204b9ef933fcd1a8b2288b3dfa49672ab765b54ee27c7" + "8a970e0e955c14f3a88e741b97c286f75f8fc299e8148362fa198a39531bed6d"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x4a000000], 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf78" + "8a3b0aa372600a92b57974cded2b9334794cba40c63e34cdea212c4cf07d41b7"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41" + "bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 1], 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32" + "111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x0706050403020100], 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c1" + "34a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a"], +] + +FSCHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "0000000000000000000000000000000000000000000000000000000000000000", 256, + "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"], + ["01", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 5, "ea"], + ["e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a", + "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", 4096, + "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"], +] + + +class TestFrameworkChacha(unittest.TestCase): + def test_chacha20(self): + """ChaCha20 test vectors.""" + for test_vector in CHACHA20_TESTS: + hex_key, nonce, counter, hex_output = test_vector + key = bytes.fromhex(hex_key) + nonce_bytes = nonce[0].to_bytes(4, 'little') + nonce[1].to_bytes(8, 'little') + keystream = chacha20_block(key, nonce_bytes, counter) + self.assertEqual(hex_output, keystream.hex()) + + def test_fschacha20(self): + """FSChaCha20 test vectors.""" + for test_vector in FSCHACHA20_TESTS: + hex_plaintext, hex_key, rekey_interval, hex_ciphertext_after_rotation = test_vector + plaintext = bytes.fromhex(hex_plaintext) + key = bytes.fromhex(hex_key) + fsc20 = FSChaCha20(key, rekey_interval) + for _ in range(rekey_interval): + fsc20.crypt(plaintext) + + ciphertext = fsc20.crypt(plaintext) + self.assertEqual(hex_ciphertext_after_rotation, ciphertext.hex()) diff --git a/test/functional/test_framework/ellswift.py b/test/functional/test_framework/crypto/ellswift.py index 97b10118e6..429b7b9f4d 100644 --- a/test/functional/test_framework/ellswift.py +++ b/test/functional/test_framework/crypto/ellswift.py @@ -12,7 +12,7 @@ import os import random import unittest -from test_framework.secp256k1 import FE, G, GE +from test_framework.crypto.secp256k1 import FE, G, GE # Precomputed constant square root of -3 (mod p). MINUS_3_SQRT = FE(-3).sqrt() diff --git a/test/functional/test_framework/ellswift_decode_test_vectors.csv b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv index 1bab96b721..1bab96b721 100644 --- a/test/functional/test_framework/ellswift_decode_test_vectors.csv +++ b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv diff --git a/test/functional/test_framework/crypto/hkdf.py b/test/functional/test_framework/crypto/hkdf.py new file mode 100644 index 0000000000..7e8958733c --- /dev/null +++ b/test/functional/test_framework/crypto/hkdf.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only HKDF-SHA256 implementation + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import hashlib +import hmac + + +def hmac_sha256(key, data): + """Compute HMAC-SHA256 from specified byte arrays key and data.""" + return hmac.new(key, data, hashlib.sha256).digest() + + +def hkdf_sha256(length, ikm, salt, info): + """Derive a key using HKDF-SHA256.""" + if len(salt) == 0: + salt = bytes([0] * 32) + prk = hmac_sha256(salt, ikm) + t = b"" + okm = b"" + for i in range((length + 32 - 1) // 32): + t = hmac_sha256(prk, t + info + bytes([i + 1])) + okm += t + return okm[:length] diff --git a/test/functional/test_framework/crypto/muhash.py b/test/functional/test_framework/crypto/muhash.py new file mode 100644 index 0000000000..09241f6203 --- /dev/null +++ b/test/functional/test_framework/crypto/muhash.py @@ -0,0 +1,55 @@ +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Native Python MuHash3072 implementation.""" + +import hashlib +import unittest + +from .chacha20 import chacha20_block + +def data_to_num3072(data): + """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" + bytes384 = b"" + for counter in range(6): + bytes384 += chacha20_block(data, bytes(12), counter) + return int.from_bytes(bytes384, 'little') + +class MuHash3072: + """Class representing the MuHash3072 computation of a set. + + See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html + """ + + MODULUS = 2**3072 - 1103717 + + def __init__(self): + """Initialize for an empty set.""" + self.numerator = 1 + self.denominator = 1 + + def insert(self, data): + """Insert a byte array data in the set.""" + data_hash = hashlib.sha256(data).digest() + self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS + + def remove(self, data): + """Remove a byte array from the set.""" + data_hash = hashlib.sha256(data).digest() + self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS + + def digest(self): + """Extract the final hash. Does not modify this object.""" + val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS + bytes384 = val.to_bytes(384, 'little') + return hashlib.sha256(bytes384).digest() + +class TestFrameworkMuhash(unittest.TestCase): + def test_muhash(self): + muhash = MuHash3072() + muhash.insert(b'\x00' * 32) + muhash.insert((b'\x01' + b'\x00' * 31)) + muhash.remove((b'\x02' + b'\x00' * 31)) + finalized = muhash.digest() + # This mirrors the result in the C++ MuHash3072 unit test + self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") diff --git a/test/functional/test_framework/crypto/poly1305.py b/test/functional/test_framework/crypto/poly1305.py new file mode 100644 index 0000000000..967b90254d --- /dev/null +++ b/test/functional/test_framework/crypto/poly1305.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of Poly1305 authenticator + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + + +class Poly1305: + """Class representing a running poly1305 computation.""" + MODULUS = 2**130 - 5 + + def __init__(self, key): + self.r = int.from_bytes(key[:16], 'little') & 0xffffffc0ffffffc0ffffffc0fffffff + self.s = int.from_bytes(key[16:], 'little') + + def tag(self, data): + """Compute the poly1305 tag.""" + acc, length = 0, len(data) + for i in range((length + 15) // 16): + chunk = data[i * 16:min(length, (i + 1) * 16)] + val = int.from_bytes(chunk, 'little') + 256**len(chunk) + acc = (self.r * (acc + val)) % Poly1305.MODULUS + return ((acc + self.s) & 0xffffffffffffffffffffffffffffffff).to_bytes(16, 'little') + + +# Test vectors from RFC7539/8439 consisting of message to be authenticated, 32 byte key and computed 16 byte tag +POLY1305_TESTS = [ + # RFC 7539, section 2.5.2. + ["43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"], + # RFC 7539, section A.3. + ["00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"], + ["2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"], + ["ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"], + ["02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"], + ["e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"], + ["e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"], +] + + +class TestFrameworkPoly1305(unittest.TestCase): + def test_poly1305(self): + """Poly1305 test vectors.""" + for test_vector in POLY1305_TESTS: + hex_message, hex_key, hex_tag = test_vector + message = bytes.fromhex(hex_message) + key = bytes.fromhex(hex_key) + tag = bytes.fromhex(hex_tag) + comp_tag = Poly1305(key).tag(message) + self.assertEqual(tag, comp_tag) diff --git a/test/functional/test_framework/ripemd160.py b/test/functional/test_framework/crypto/ripemd160.py index 12801364b4..12801364b4 100644 --- a/test/functional/test_framework/ripemd160.py +++ b/test/functional/test_framework/crypto/ripemd160.py diff --git a/test/functional/test_framework/secp256k1.py b/test/functional/test_framework/crypto/secp256k1.py index 2e9e419da5..2e9e419da5 100644 --- a/test/functional/test_framework/secp256k1.py +++ b/test/functional/test_framework/crypto/secp256k1.py diff --git a/test/functional/test_framework/siphash.py b/test/functional/test_framework/crypto/siphash.py index bd13b2c948..bd13b2c948 100644 --- a/test/functional/test_framework/siphash.py +++ b/test/functional/test_framework/crypto/siphash.py diff --git a/test/functional/test_framework/xswiftec_inv_test_vectors.csv b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv index 138c4cf85c..138c4cf85c 100644 --- a/test/functional/test_framework/xswiftec_inv_test_vectors.csv +++ b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 6c1892539f..06252f8996 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -13,7 +13,7 @@ import os import random import unittest -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 # Point with no known discrete log. H_POINT = "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0" diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8f3aea8785..d008cb39aa 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -29,7 +29,7 @@ import struct import time import unittest -from test_framework.siphash import siphash256 +from test_framework.crypto.siphash import siphash256 from test_framework.util import assert_equal MAX_LOCATOR_SZ = 101 diff --git a/test/functional/test_framework/muhash.py b/test/functional/test_framework/muhash.py deleted file mode 100644 index 0d96114e3e..0000000000 --- a/test/functional/test_framework/muhash.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2020 Pieter Wuille -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Native Python MuHash3072 implementation.""" - -import hashlib -import unittest - -def rot32(v, bits): - """Rotate the 32-bit value v left by bits bits.""" - bits %= 32 # Make sure the term below does not throw an exception - return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) - -def chacha20_doubleround(s): - """Apply a ChaCha20 double round to 16-element state array s. - - See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 - """ - QUARTER_ROUNDS = [(0, 4, 8, 12), - (1, 5, 9, 13), - (2, 6, 10, 14), - (3, 7, 11, 15), - (0, 5, 10, 15), - (1, 6, 11, 12), - (2, 7, 8, 13), - (3, 4, 9, 14)] - - for a, b, c, d in QUARTER_ROUNDS: - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 16) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 12) - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 8) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 7) - -def chacha20_32_to_384(key32): - """Specialized ChaCha20 implementation with 32-byte key, 0 IV, 384-byte output.""" - # See RFC 8439 section 2.3 for chacha20 parameters - CONSTANTS = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574] - - key_bytes = [0]*8 - for i in range(8): - key_bytes[i] = int.from_bytes(key32[(4 * i):(4 * (i+1))], 'little') - - INITIALIZATION_VECTOR = [0] * 4 - init = CONSTANTS + key_bytes + INITIALIZATION_VECTOR - out = bytearray() - for counter in range(6): - init[12] = counter - s = init.copy() - for _ in range(10): - chacha20_doubleround(s) - for i in range(16): - out.extend(((s[i] + init[i]) & 0xffffffff).to_bytes(4, 'little')) - return bytes(out) - -def data_to_num3072(data): - """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" - bytes384 = chacha20_32_to_384(data) - return int.from_bytes(bytes384, 'little') - -class MuHash3072: - """Class representing the MuHash3072 computation of a set. - - See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html - """ - - MODULUS = 2**3072 - 1103717 - - def __init__(self): - """Initialize for an empty set.""" - self.numerator = 1 - self.denominator = 1 - - def insert(self, data): - """Insert a byte array data in the set.""" - data_hash = hashlib.sha256(data).digest() - self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS - - def remove(self, data): - """Remove a byte array from the set.""" - data_hash = hashlib.sha256(data).digest() - self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS - - def digest(self): - """Extract the final hash. Does not modify this object.""" - val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS - bytes384 = val.to_bytes(384, 'little') - return hashlib.sha256(bytes384).digest() - -class TestFrameworkMuhash(unittest.TestCase): - def test_muhash(self): - muhash = MuHash3072() - muhash.insert(b'\x00' * 32) - muhash.insert((b'\x01' + b'\x00' * 31)) - muhash.remove((b'\x02' + b'\x00' * 31)) - finalized = muhash.digest() - # This mirrors the result in the C++ MuHash3072 unit test - self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") - - def test_chacha20(self): - def chacha_check(key, result): - self.assertEqual(chacha20_32_to_384(key)[:64].hex(), result) - - # Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 - # Since the nonce is hardcoded to 0 in our function we only use those vectors. - chacha_check([0]*32, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586") - chacha_check([0]*31 + [1], "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963") diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index be4ed624fc..b1ed97b794 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -364,8 +364,8 @@ class P2PInterface(P2PConnection): vt.addrFrom.port = 0 self.on_connection_send_msg = vt # Will be sent in connection_made callback - def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs): - create_conn = super().peer_connect(*args, **kwargs) + def peer_connect(self, *, services=P2P_SERVICES, send_version, **kwargs): + create_conn = super().peer_connect(**kwargs) if send_version: self.peer_connect_send_version(services) @@ -456,7 +456,8 @@ class P2PInterface(P2PConnection): self.send_message(msg_verack()) self.nServices = message.nServices self.relay = message.relay - self.send_message(msg_getaddr()) + if self.p2p_connected_to_node: + self.send_message(msg_getaddr()) # Connection helper methods diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 17a954cb22..f4628bf4af 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -24,7 +24,7 @@ from .messages import ( uint256_from_str, ) -from .ripemd160 import ripemd160 +from .crypto.ripemd160 import ripemd160 MAX_SCRIPT_ELEMENT_SIZE = 520 MAX_PUBKEYS_PER_MULTI_A = 999 diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c77cfbdd91..281469f9ed 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -634,7 +634,7 @@ class TestNode(): assert_msg += "with expected error " + expected_msg self._raise_assertion_error(assert_msg) - def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): + def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, **kwargs): """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also @@ -644,9 +644,12 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' - p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() + p2p_conn.p2p_connected_to_node = True + p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False) + if send_version: + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: # Wait for the node to send us the version and verack p2p_conn.wait_for_verack() @@ -689,6 +692,7 @@ class TestNode(): self.log.debug("Connecting to %s:%d %s" % (address, port, connection_type)) self.addconnection('%s:%d' % (address, port), connection_type) + p2p_conn.p2p_connected_to_node = False p2p_conn.peer_accept_connection(connect_cb=addconnection_callback, connect_id=p2p_idx + 1, net=self.chain, timeout_factor=self.timeout_factor, **kwargs)() if connection_type == "feeler": @@ -699,6 +703,7 @@ class TestNode(): p2p_conn.wait_for_connect() self.p2ps.append(p2p_conn) + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: p2p_conn.wait_for_verack() p2p_conn.sync_with_ping() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 341a31909b..35c662b0d9 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -73,12 +73,15 @@ TEST_EXIT_SKIPPED = 77 # the output of `git grep unittest.TestCase ./test/functional/test_framework` TEST_FRAMEWORK_MODULES = [ "address", + "crypto.bip324_cipher", "blocktools", - "ellswift", + "crypto.chacha20", + "crypto.ellswift", "key", "messages", - "muhash", - "ripemd160", + "crypto.muhash", + "crypto.poly1305", + "crypto.ripemd160", "script", "segwit_addr", ] @@ -105,7 +108,6 @@ BASE_SCRIPTS = [ 'feature_maxuploadtarget.py', 'mempool_updatefromblock.py', 'mempool_persist.py --descriptors', - 'wallet_miniscript.py --descriptors', # vv Tests less than 60s vv 'rpc_psbt.py --legacy-wallet', 'rpc_psbt.py --descriptors', @@ -149,6 +151,7 @@ BASE_SCRIPTS = [ 'p2p_sendheaders.py', 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', + 'wallet_miniscript.py --descriptors', # vv Tests less than 30s vv 'p2p_invalid_messages.py', 'rpc_createmultisig.py', diff --git a/test/functional/wallet_miniscript.py b/test/functional/wallet_miniscript.py index d174b525b3..67e1283902 100755 --- a/test/functional/wallet_miniscript.py +++ b/test/functional/wallet_miniscript.py @@ -205,10 +205,10 @@ DESCS_PRIV = [ class WalletMiniscriptTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser, legacy=False) - self.rpc_timeout = 480 def set_test_params(self): self.num_nodes = 1 + self.rpc_timeout = 180 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index ec74f7705c..e72977fac0 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -16,11 +16,14 @@ import sys def get_fuzz_env(*, target, source_dir): + symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") return { 'FUZZ': target, 'UBSAN_OPTIONS': f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'UBSAN_SYMBOLIZER_PATH':symbolizer, "ASAN_OPTIONS": "detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", + 'ASAN_SYMBOLIZER_PATH':symbolizer, } diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index d22dd9d996..f55d0f8cb7 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb', '-zapwallettxes']) +SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def lint_missing_argument_documentation(): diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index e366a08bd2..6f9a633807 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -21,7 +21,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES = ( "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", "wallet/wallet -> wallet/walletdb -> wallet/wallet", "kernel/coinstats -> validation -> kernel/coinstats", - "kernel/mempool_persist -> validation -> kernel/mempool_persist", # Temporary, removed in followup https://github.com/bitcoin/bitcoin/pull/24230 "index/base -> node/context -> net_processing -> index/blockfilterindex -> index/base", diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 8e79ba5121..6386a92c0f 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -23,6 +23,7 @@ EXCLUDED_DIRS = ["contrib/devtools/bitcoin-tidy/", ] EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", + "boost/multi_index/detail/hash_index_iterator.hpp", "boost/multi_index/hashed_index.hpp", "boost/multi_index/identity.hpp", "boost/multi_index/indexed_by.hpp", @@ -30,6 +31,7 @@ EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/sequenced_index.hpp", "boost/multi_index/tag.hpp", "boost/multi_index_container.hpp", + "boost/operators.hpp", "boost/process.hpp", "boost/signals2/connection.hpp", "boost/signals2/optional_last_value.hpp", |