diff options
186 files changed, 2514 insertions, 1352 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 3f52adde8a..5c5942e2d5 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -84,6 +84,9 @@ task: memory: 1G # For faster CI feedback, immediately schedule the linters << : *CREDITS_TEMPLATE + test_runner_cache: + folder: "/lint_test_runner" + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner) python_cache: folder: "/python_build" fingerprint_script: cat .python-version /etc/os-release @@ -111,7 +114,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64, unit tests, no gui tests, no boost::process, no functional tests' + name: 'Win64, unit tests, no gui tests, no functional tests' << : *GLOBAL_TASK_TEMPLATE persistent_worker: labels: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5939520ad7..1cd40be541 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: name: 'macOS 13 native, x86_64, no depends, sqlite only, gui' # Use latest image, but hardcode version to avoid silent upgrades (and breaks). # See: https://github.com/actions/runner-images#available-images. - runs-on: macos-13 # Use M1 once available https://github.com/github/roadmap/issues/528 + runs-on: macos-13 # No need to run on the read-only mirror, unless it is a PR. if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request' diff --git a/Makefile.am b/Makefile.am index 7c3cd82e7f..c50421dfc3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,8 +14,6 @@ endif .PHONY: deploy FORCE .INTERMEDIATE: $(COVERAGE_INFO) -export PYTHONPATH - if BUILD_BITCOIN_LIBS pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libbitcoinconsensus.pc @@ -189,7 +187,7 @@ $(COV_TOOL_WRAPPER): @chmod +x $(COV_TOOL_WRAPPER) baseline.info: $(COV_TOOL_WRAPPER) - $(LCOV) -c -i -d $(abs_builddir)/src -o $@ + $(LCOV) $(LCOV_OPTS) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info $(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@ @@ -223,13 +221,13 @@ functional_test_filtered.info: functional_test.info $(LCOV) -a $@ $(LCOV_OPTS) -o $@ fuzz_coverage.info: fuzz_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin_coverage.info: baseline_filtered.info test_bitcoin_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -o $@ + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -o $@ total_coverage.info: test_bitcoin_filtered.info functional_test_filtered.info - $(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt + $(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt fuzz.coverage/.dirstamp: fuzz_coverage.info $(GENHTML) -s $(LCOV_OPTS) $< -o $(@D) diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 602b57fe43..5e65257d77 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -4,8 +4,10 @@ dnl permitted in any medium without royalty provided the copyright notice dnl and this notice are preserved. This file is offered as-is, without any dnl warranty. -# Some versions of gcc/libstdc++ require linking with -latomic if -# using the C++ atomic library. +# Clang prior to version 15, when building for 32-bit, +# and linking against libstdc++, requires linking with +# -latomic if using the C++ atomic library. +# Can be tested with: clang++ test.cpp -m32 # # Sourced from http://bugs.debian.org/797228 diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index b160406392..476417d04b 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -33,6 +33,17 @@ export PATH="${PYTHON_PATH}/bin:${PATH}" command -v python3 python3 --version +export LINT_RUNNER_PATH="/lint_test_runner" +if [ ! -d "${LINT_RUNNER_PATH}" ]; then + ${CI_RETRY_EXE} apt-get install -y cargo + ( + cd ./test/lint/test_runner || exit 1 + cargo build + mkdir -p "${LINT_RUNNER_PATH}" + mv target/debug/test_runner "${LINT_RUNNER_PATH}" + ) +fi + ${CI_RETRY_EXE} pip3 install \ codespell==2.2.5 \ flake8==6.1.0 \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index ccde12a033..af7a517930 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -30,6 +30,7 @@ test/lint/git-subtree-check.sh src/secp256k1 test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c +RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" test/lint/check-doc.py test/lint/all-lint.py diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh index e94a75e22c..a403f923a2 100755 --- a/ci/lint/container-entrypoint.sh +++ b/ci/lint/container-entrypoint.sh @@ -11,6 +11,7 @@ export LC_ALL=C git config --global --add safe.directory /bitcoin export PATH="/python_build/bin:${PATH}" +export LINT_RUNNER_PATH="/lint_test_runner" if [ -z "$1" ]; then LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index c9f65bf397..439fba16ef 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -15,4 +15,3 @@ export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_MAXSIZE=400M export RUN_FUZZ_TESTS=true -export FUZZ_TESTS_CONFIG="--exclude banman" # https://github.com/bitcoin/bitcoin/issues/27924 diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 122f044b58..3585b2b417 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -18,3 +18,4 @@ export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ CC='clang-17 -ftrivial-auto-var-init=pattern' CXX='clang++-17 -ftrivial-auto-var-init=pattern'" export CCACHE_MAXSIZE=200M +export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-17" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index a0b054ab40..b15df4b6cc 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -67,8 +67,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then fi if [[ "${RUN_TIDY}" == "true" ]]; then - ${CI_RETRY_EXE} git clone https://github.com/include-what-you-use/include-what-you-use -b master /include-what-you-use - git -C /include-what-you-use checkout a138eaac254e5a472464e31d5ec418fe6e6f1fc7 + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use make -C /iwyu-build/ install "-j$( nproc )" # Use nproc, because MAKEJOBS is the default in docker image builds fi diff --git a/configure.ac b/configure.ac index 6add570d00..20992380d2 100644 --- a/configure.ac +++ b/configure.ac @@ -139,8 +139,6 @@ AC_PATH_TOOL([OBJCOPY], [objcopy]) AC_PATH_PROG([DOXYGEN], [doxygen]) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) -AC_ARG_VAR([PYTHONPATH], [Augments the default search path for python module files]) - AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], [disable wallet (enabled by default)])], @@ -1002,7 +1000,6 @@ dnl "ad_strip" as the symbol for the entry point. if test "$TARGET_OS" = "darwin"; then AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-fixup_chains], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-fixup_chains"], [], [$LDFLAG_WERROR]) fi @@ -1970,6 +1967,17 @@ case ${OS} in ;; esac +dnl An old hack similar to a98356fee to remove hard-coded +dnl bind_at_load flag from libtool +case $host in + *darwin*) + AC_MSG_RESULT([Removing -Wl,bind_at_load from libtool.]) + sed < libtool > libtool-2 '/bind_at_load/d' + mv libtool-2 libtool + chmod 755 libtool + ;; +esac + echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 802bf9fd30..92272cf8f9 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -129,11 +129,11 @@ class TestSecurityChecks(unittest.TestCase): (1, executable+': failed NOUNDEFS PIE CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-Wl,-fixup_chains']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']), (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']), (0, '')) else: # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks @@ -143,7 +143,7 @@ class TestSecurityChecks(unittest.TestCase): (1, executable+': failed NOUNDEFS Canary')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']), (1, executable+': failed NOUNDEFS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all', '-Wl,-fixup_chains']), + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']), (0, '')) diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 4d5b06a54d..28ca23098d 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=160f78a4d92205df986ed9efcce7d3aac188cb24 \ + --commit=77386bdbfe6b0c649c05ab37f08051d1ab3e5074 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 12f57fcc45..7a030e5eab 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -11,7 +11,7 @@ (gnu packages gawk) (gnu packages gcc) ((gnu packages installers) #:select (nsis-x86_64)) - ((gnu packages linux) #:select (linux-libre-headers-5.15 util-linux)) + ((gnu packages linux) #:select (linux-libre-headers-6.1 util-linux)) (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) @@ -92,7 +92,7 @@ chain for " target " development.")) (license (package-license xgcc))))) (define base-gcc gcc-10) -(define base-linux-kernel-headers linux-libre-headers-5.15) +(define base-linux-kernel-headers linux-libre-headers-6.1) (define* (make-bitcoin-cross-toolchain target #:key @@ -372,79 +372,8 @@ certificates or paths. Supports various options, including: validation at a specific moment in time, whitelisting and revocation checks.") (license license:expat)))) -(define-public python-altgraph - (package - (name "python-altgraph") - (version "0.17") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/ronaldoussoren/altgraph") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "09sm4srvvkw458pn48ga9q7ykr4xlz7q8gh1h9w7nxpf001qgpwb")))) - (build-system python-build-system) - (home-page "https://github.com/ronaldoussoren/altgraph") - (synopsis "Python graph (network) package") - (description "altgraph is a fork of graphlib: a graph (network) package for -constructing graphs, BFS and DFS traversals, topological sort, shortest paths, -etc. with graphviz output.") - (license license:expat))) - - -(define-public python-macholib - (package - (name "python-macholib") - (version "1.14") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/ronaldoussoren/macholib") - (commit (string-append "v" version)))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "0aislnnfsza9wl4f0vp45ivzlc0pzhp9d4r08700slrypn5flg42")))) - (build-system python-build-system) - (propagated-inputs - `(("python-altgraph" ,python-altgraph))) - (arguments - '(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - ;; This test is broken as there is no keyboard interrupt. - (substitute* "macholib_tests/test_command_line.py" - (("^(.*)class TestCmdLine" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "macholib_tests/test_dyld.py" - (("^(.*)def test_\\S+_find" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line)) - (("^(.*)def testBasic" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line)) - ) - #t))))) - (home-page "https://github.com/ronaldoussoren/macholib") - (synopsis "Python library for analyzing and editing Mach-O headers") - (description "macholib is a Macho-O header analyzer and editor. It's -typically used as a dependency analysis tool, and also to rewrite dylib -references in Mach-O headers to be @executable_path relative. Though this tool -targets a platform specific file format, it is pure python code that is platform -and endian independent.") - (license license:expat))) - (define-public python-signapple - (let ((commit "7a96b4171a360abf0f0f56e499f8f9ed2116280d")) + (let ((commit "62155712e7417aba07565c9780a80e452823ae6a")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -457,14 +386,13 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "0aa4k180jnpal15yhncnm3g3z9gzmi7qb25q5l0kaj444a1p2pm4")))) + "1nm6rm4h4m7kbq729si4cm8rzild62mk4ni8xr5zja7l33fhv3gb")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) ("python-oscrypto" ,python-oscrypto) ("python-certvalidator" ,python-certvalidator) - ("python-elfesteem" ,python-elfesteem) - ("python-macholib" ,python-macholib))) + ("python-elfesteem" ,python-elfesteem))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) diff --git a/contrib/signet/miner b/contrib/signet/miner index 61d9f62be7..e5daf9f993 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -30,7 +30,7 @@ logging.basicConfig( SIGNET_HEADER = b"\xec\xc7\xda\xa2" PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed -RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") +RE_MULTIMINER = re.compile(r"^(\d+)(-(\d+))?/(\d+)$") def create_coinbase(height, value, spk): cb = CTransaction() diff --git a/depends/config.guess b/depends/config.guess index 69188da73d..cdfc439204 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -4,7 +4,7 @@ # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2023-01-01' +timestamp='2023-08-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -47,7 +47,7 @@ me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -66,7 +66,7 @@ This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -102,8 +102,8 @@ GUESS= # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -155,6 +155,9 @@ Linux|GNU|GNU/*) set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include <features.h> #if defined(__UCLIBC__) LIBC=uclibc @@ -169,6 +172,7 @@ Linux|GNU|GNU/*) LIBC=musl #endif #endif + #endif EOF cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` eval "$cc_set_libc" @@ -459,7 +463,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. + # Japanese Language versions have a version number like '4.1.3-JL'. SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` GUESS=sparc-sun-sunos$SUN_REL ;; @@ -904,7 +908,7 @@ EOF fi ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; @@ -976,7 +980,27 @@ EOF GUESS=$UNAME_MACHINE-unknown-minix ;; aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be @@ -1042,6 +1066,15 @@ EOF k1om:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; loongarch32:Linux:*:* | loongarch64:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; @@ -1197,7 +1230,7 @@ EOF GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. GUESS=$UNAME_MACHINE-pc-os2-emx ;; @@ -1338,7 +1371,7 @@ EOF GUESS=ns32k-sni-sysv fi ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says <Richard.M.Bartel@ccMail.Census.GOV> GUESS=i586-unisys-sysv4 ;; diff --git a/depends/config.site.in b/depends/config.site.in index 05c2ccbac1..a6a5596bb8 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -102,7 +102,6 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}" if test -n "@AR@"; then AR="@AR@" diff --git a/depends/config.sub b/depends/config.sub index de4259e404..defe52c0c8 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -4,7 +4,7 @@ # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2023-01-21' +timestamp='2023-09-19' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -82,7 +82,7 @@ This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -130,7 +130,7 @@ IFS=$saved_IFS # Separate into logical components for further validation case $1 in *-*-*-*-*) - echo Invalid configuration \`"$1"\': more than four components >&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) @@ -145,7 +145,8 @@ case $1 in nto-qnx* | linux-* | uclinux-uclibc* \ | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova* | managarm-*) + | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + | windows-* ) basic_machine=$field1 basic_os=$maybe_os ;; @@ -943,7 +944,7 @@ $basic_machine EOF IFS=$saved_IFS ;; - # We use `pc' rather than `unknown' + # We use 'pc' rather than 'unknown' # because (1) that's what they normally are, and # (2) the word "unknown" tends to confuse beginning users. i*86 | x86_64) @@ -1180,7 +1181,7 @@ case $cpu-$vendor in case $cpu in 1750a | 580 \ | a29k \ - | aarch64 | aarch64_be \ + | aarch64 | aarch64_be | aarch64c | arm64ec \ | abacus \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ @@ -1199,12 +1200,14 @@ case $cpu-$vendor in | d10v | d30v | dlx | dsp16xx \ | e2k | elxsi | epiphany \ | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | javascript \ | h8300 | h8500 \ | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ | i370 | i*86 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ + | kvx \ | le32 | le64 \ | lm32 \ | loongarch32 | loongarch64 \ @@ -1213,31 +1216,7 @@ case $cpu-$vendor in | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ | m88110 | m88k | maxq | mb | mcore | mep | metag \ | microblaze | microblazeel \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64eb | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r3 | mipsisa32r3el \ - | mipsisa32r5 | mipsisa32r5el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r3 | mipsisa64r3el \ - | mipsisa64r5 | mipsisa64r5el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ + | mips* \ | mmix \ | mn10200 | mn10300 \ | moxie \ @@ -1285,7 +1264,7 @@ case $cpu-$vendor in ;; *) - echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1306,11 +1285,12 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if test x$basic_os != x +if test x"$basic_os" != x then # First recognize some ad-hoc cases, or perhaps split kernel-os, or else just # set os. +obj= case $basic_os in gnu/linux*) kernel=linux @@ -1510,10 +1490,16 @@ case $os in os=eabi ;; *) - os=elf + os= + obj=elf ;; esac ;; + aout* | coff* | elf* | pe*) + # These are machine code file formats, not OSes + obj=$os + os= + ;; *) # No normalization, but not necessarily accepted, that comes below. ;; @@ -1532,12 +1518,15 @@ else # system, and we'll never get to this point. kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 @@ -1547,28 +1536,35 @@ case $cpu-$vendor in os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1590,19 +1586,24 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf ;; mips*-cisco) - os=elf + os= + obj=elf ;; mips*-*) - os=elf + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; *-tti) # must be before sparc entry or we get the wrong os. os=sysv3 @@ -1611,7 +1612,8 @@ case $cpu-$vendor in os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1692,10 +1694,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1713,7 +1717,8 @@ esac fi -# Now, validate our (potentially fixed-up) OS. +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + case $os in # Sometimes we do "kernel-libc", so those need to count as OSes. musl* | newlib* | relibc* | uclibc*) @@ -1724,6 +1729,9 @@ case $os in # VxWorks passes extra cpu info in the 4th filed. simlinux | simwindows | spe) ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1732,7 +1740,7 @@ case $os in | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* \ + | os9* | macos* | osx* | ios* | tvos* | watchos* \ | mpw* | magic* | mmixware* | mon960* | lnews* \ | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ | aos* | aros* | cloudabi* | sortix* | twizzler* \ @@ -1741,11 +1749,11 @@ case $os in | mirbsd* | netbsd* | dicos* | openedition* | ose* \ | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ - | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | bosx* | nextstep* | cxux* | oabi* \ + | ptx* | ecoff* | winnt* | domain* | vsta* \ | udi* | lites* | ieee* | go32* | aux* | hcos* \ | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | cygwin* | msys* | moss* | proelf* | rtems* \ | midipix* | mingw32* | mingw64* | mint* \ | uxpv* | beos* | mpeix* | udk* | moxiebox* \ | interix* | uwin* | mks* | rhapsody* | darwin* \ @@ -1758,7 +1766,7 @@ case $os in | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ - | fiwix* | mlibc* ) + | fiwix* | mlibc* | cos* | mbr* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) @@ -1766,54 +1774,99 @@ case $os in ;; none) ;; - kernel* ) + kernel* | msvc* ) # Restricted further below ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; + *) + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; *) - echo Invalid configuration \`"$1"\': OS \`"$os"\' not recognized 1>&2 + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 exit 1 ;; esac # As a final step for OS-related things, validate the OS-kernel combination # (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* \ - | linux-musl* | linux-relibc* | linux-uclibc* | linux-mlibc* ) +case $kernel-$os-$obj in + linux-gnu*- | linux-dietlibc*- | linux-android*- | linux-newlib*- \ + | linux-musl*- | linux-relibc*- | linux-uclibc*- | linux-mlibc*- ) ;; - uclinux-uclibc* ) + uclinux-uclibc*- ) ;; - managarm-mlibc* | managarm-kernel* ) + managarm-mlibc*- | managarm-kernel*- ) ;; - -dietlibc* | -newlib* | -musl* | -relibc* | -uclibc* | -mlibc* ) + windows*-msvc*-) + ;; + -dietlibc*- | -newlib*- | -musl*- | -relibc*- | -uclibc*- | -mlibc*- ) # These are just libc implementations, not actual OSes, and thus # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 exit 1 ;; - -kernel* ) - echo "Invalid configuration \`$1': \`$os' needs explicit kernel." 1>&2 + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 exit 1 ;; - *-kernel* ) - echo "Invalid configuration \`$1': \`$kernel' does not support \`$os'." 1>&2 + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 exit 1 ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) + kfreebsd*-gnu*- | kopensolaris*-gnu*-) ;; - nto-qnx*) + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; - os2-emx) + nto-qnx*-) + ;; + os2-emx-) + ;; + *-eabi*- | *-gnueabi*-) ;; - *-eabi* | *-gnueabi*) + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format ;; - -*) + -*-) # Blank kernel with real OS is always fine. ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 exit 1 ;; esac @@ -1896,7 +1949,7 @@ case $vendor in ;; esac -echo "$cpu-$vendor-${kernel:+$kernel-}$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/depends/description.md b/depends/description.md index ef6e6df183..69ee5bd36c 100644 --- a/depends/description.md +++ b/depends/description.md @@ -6,8 +6,7 @@ There are several features that make it different from most similar systems: In theory, binaries for any target OS/architecture can be created, from a builder running any OS/architecture. In practice, build-side tools must be specified when the defaults don't fit, and packages must be amended to work -on new hosts. For now, a build architecture of x86_64 is assumed, either on -Linux or macOS. +on new hosts. ### No reliance on timestamps diff --git a/doc/build-osx.md b/doc/build-osx.md index 76fc357956..5c3dc1ac7f 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -175,7 +175,6 @@ There are many ways to configure Bitcoin Core, here are a few common examples: ##### Wallet (BDB + SQlite) Support, No GUI: If `berkeley-db@4` is installed, then legacy wallet support will be built. -If `berkeley-db@4` is not installed, then this will throw an error. If `sqlite` is installed, then descriptor wallet support will also be built. Additionally, this explicitly disables the GUI. diff --git a/doc/design/multiprocess.md b/doc/design/multiprocess.md index e6a77dbbc0..45681d12de 100644 --- a/doc/design/multiprocess.md +++ b/doc/design/multiprocess.md @@ -19,7 +19,7 @@ The `-debug=ipc` command line option can be used to see requests and responses b ## Installation -The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: +The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../../depends) with the `MULTIPROCESS=1` [dependency option](../../depends#dependency-options) passed to make: ``` cd <BITCOIN_SOURCE_DIRECTORY> @@ -32,12 +32,12 @@ BITCOIND=bitcoin-node test/functional/test_runner.py The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). -Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](../build-unix.md) and [build-osx.md](../build-osx.md) for information about installing dependencies in general. ## IPC implementation details Cross process Node, Wallet, and Chain interfaces are defined in -[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow +[`src/interfaces/`](../../src/interfaces/). These are C++ classes which follow [conventions](../developer-notes.md#internal-interface-guidelines), like passing serializable arguments so they can be called from different processes, and making methods pure virtual so they can have proxy implementations that forward @@ -58,7 +58,7 @@ actual serialization and socket communication. As much as possible, calls between processes are meant to work the same as calls within a single process without adding limitations or requiring extra implementation effort. Processes communicate with each other by calling regular -[C++ interface methods](../src/interfaces/README.md). Method arguments and +[C++ interface methods](../../src/interfaces/README.md). Method arguments and return values are automatically serialized and sent between processes. Object references and `std::function` arguments are automatically tracked and mapped to allow invoked code to call back into invoking code at any time, and there is diff --git a/doc/release-notes-28207.md b/doc/release-notes-28207.md new file mode 100644 index 0000000000..56b88da16a --- /dev/null +++ b/doc/release-notes-28207.md @@ -0,0 +1,7 @@ +mempool.dat compatibility +======================== + +The `mempool.dat` file created by -persistmempool or the savemempool RPC will +be written in a new format, which can not be read by previous software +releases. To allow for a downgrade, a temporary setting `-persistmempoolv1` has +been added to fall back to the legacy format. diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 4a42557372..e9d3bff1de 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -112,6 +112,7 @@ BITCOIN_TESTS =\ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ + test/net_peer_connection_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ @@ -145,6 +146,7 @@ BITCOIN_TESTS =\ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/sock_tests.cpp \ + test/span_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 269ac847a5..9f52a22672 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -24,7 +24,7 @@ static void DeserializeBlockTest(benchmark::Bench& bench) bench.unit("block").run([&] { CBlock block; - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); }); @@ -41,7 +41,7 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) bench.unit("block").run([&] { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 8b3dc6c71c..127240d3c7 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 10 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 11 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -120,6 +120,10 @@ # define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif +// noexcept may be missing for std::string. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58265 +#define ANKERL_NANOBENCH_PRIVATE_NOEXCEPT_STRING_MOVE() std::is_nothrow_move_assignable<std::string>::value + // declarations /////////////////////////////////////////////////////////////////////////////////// namespace ankerl { @@ -404,7 +408,7 @@ struct Config { Config(); ~Config(); Config& operator=(Config const& other); - Config& operator=(Config&& other) noexcept; + Config& operator=(Config&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Config(Config const& other); Config(Config&& other) noexcept; }; @@ -430,7 +434,7 @@ public: ~Result(); Result& operator=(Result const& other); - Result& operator=(Result&& other) noexcept; + Result& operator=(Result&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Result(Result const& other); Result(Result&& other) noexcept; @@ -596,7 +600,7 @@ public: * * @return Vector containing the full state: */ - std::vector<uint64_t> state() const; + ANKERL_NANOBENCH(NODISCARD) std::vector<uint64_t> state() const; private: static constexpr uint64_t rotl(uint64_t x, unsigned k) noexcept; @@ -628,7 +632,7 @@ public: Bench(); Bench(Bench&& other) noexcept; - Bench& operator=(Bench&& other) noexcept; + Bench& operator=(Bench&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Bench(Bench const& other); Bench& operator=(Bench const& other); ~Bench() noexcept; @@ -818,7 +822,7 @@ public: * Default is zero, so we are fully relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see * that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations(). * - * @see maxEpochTim), minEpochIterations + * @see maxEpochTime, minEpochIterations * * @param t Minimum time each epoch should take. */ @@ -1030,7 +1034,7 @@ void doNotOptimizeAway(T const& val); // These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but // this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L443-L446 template <typename T> void doNotOptimizeAway(T const& val) { // NOLINTNEXTLINE(hicpp-no-assembler) @@ -1781,7 +1785,7 @@ bool isEndlessRunning(std::string const& name); bool isWarningsEnabled(); template <typename T> -T parseFile(std::string const& filename); +T parseFile(std::string const& filename, bool* fail); void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector<std::string>& recommendations); void printStabilityInformationOnce(std::ostream* outStream); @@ -1839,7 +1843,7 @@ class Number { public: Number(int width, int precision, double value); Number(int width, int precision, int64_t value); - std::string to_s() const; + ANKERL_NANOBENCH(NODISCARD) std::string to_s() const; private: friend std::ostream& operator<<(std::ostream& os, Number const& n); @@ -1857,11 +1861,11 @@ std::ostream& operator<<(std::ostream& os, Number const& n); class MarkDownColumn { public: - MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val); - std::string title() const; - std::string separator() const; - std::string invalid() const; - std::string value() const; + MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string title() const; + ANKERL_NANOBENCH(NODISCARD) std::string separator() const; + ANKERL_NANOBENCH(NODISCARD) std::string invalid() const; + ANKERL_NANOBENCH(NODISCARD) std::string value() const; private: int mWidth; @@ -1976,9 +1980,9 @@ PerformanceCounters& performanceCounters() { } // Windows version of doNotOptimizeAway -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 -// see https://github.com/facebook/folly/blob/master/folly/Benchmark.h#L280 -// see https://docs.microsoft.com/en-us/cpp/preprocessor/optimize +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L514 +// see https://github.com/facebook/folly/blob/v2023.01.30.00/folly/lang/Hint-inl.h#L54-L58 +// see https://learn.microsoft.com/en-us/cpp/preprocessor/optimize # if defined(_MSC_VER) # pragma optimize("", off) void doNotOptimizeAwaySink(void const*) {} @@ -1986,10 +1990,13 @@ void doNotOptimizeAwaySink(void const*) {} # endif template <typename T> -T parseFile(std::string const& filename) { +T parseFile(std::string const& filename, bool* fail) { std::ifstream fin(filename); // NOLINT(misc-const-correctness) T num{}; fin >> num; + if (fail != nullptr) { + *fail = fin.fail(); + } return num; } @@ -2032,16 +2039,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< if (nprocs <= 0) { warnings.emplace_back("couldn't figure out number of processors - no governor, turbo check possible"); } else { - // check frequency scaling for (long id = 0; id < nprocs; ++id) { auto idStr = detail::fmt::to_s(static_cast<uint64_t>(id)); auto sysCpu = "/sys/devices/system/cpu/cpu" + idStr; - auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq"); - auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq"); + auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq", nullptr); + auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq", nullptr); if (minFreq != maxFreq) { - auto minMHz = static_cast<double>(minFreq) / 1000.0; - auto maxMHz = static_cast<double>(maxFreq) / 1000.0; + auto minMHz = d(minFreq) / 1000.0; + auto maxMHz = d(maxFreq) / 1000.0; warnings.emplace_back("CPU frequency scaling enabled: CPU " + idStr + " between " + detail::fmt::Number(1, 1, minMHz).to_s() + " and " + detail::fmt::Number(1, 1, maxMHz).to_s() + " MHz"); @@ -2050,13 +2056,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< } } - auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); - if ("performance" != currentGovernor) { + auto fail = false; + auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", &fail); + if (!fail && "performance" != currentGovernor) { warnings.emplace_back("CPU governor is '" + currentGovernor + "' but should be 'performance'"); recommendPyPerf = true; } - if (0 == parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo")) { + auto noTurbo = parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo", &fail); + if (!fail && noTurbo == 0) { warnings.emplace_back("Turbo is enabled, CPU frequency will fluctuate"); recommendPyPerf = true; } @@ -2250,10 +2258,9 @@ struct IterationLogic::Impl { mNumIters = 0; } - ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, static_cast<double>(elapsed.count())) << " elapsed, " - << detail::fmt::Number(20, 3, static_cast<double>(mTargetRuntimePerEpoch.count())) - << " target. oldIters=" << oldIters << ", mNumIters=" << mNumIters - << ", mState=" << static_cast<int>(mState)); + ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, d(elapsed.count())) << " elapsed, " + << detail::fmt::Number(20, 3, d(mTargetRuntimePerEpoch.count())) << " target. oldIters=" + << oldIters << ", mNumIters=" << mNumIters << ", mState=" << static_cast<int>(mState)); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -2357,7 +2364,7 @@ struct IterationLogic::Impl { } os << fmt::MarkDownCode(mBench.name()); if (showUnstable) { - auto avgIters = static_cast<double>(mTotalNumIters) / static_cast<double>(mBench.epochs()); + auto avgIters = d(mTotalNumIters) / d(mBench.epochs()); // NOLINTNEXTLINE(bugprone-incorrect-roundings) auto suggestedIters = static_cast<uint64_t>(avgIters * 10 + 0.5); @@ -2435,7 +2442,7 @@ public: bool monitor(perf_sw_ids swId, Target target); bool monitor(perf_hw_id hwId, Target target); - bool hasError() const noexcept { + ANKERL_NANOBENCH(NODISCARD) bool hasError() const noexcept { return mHasError; } @@ -2691,16 +2698,23 @@ PerformanceCounters::PerformanceCounters() , mVal() , mHas() { - mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + // HW events mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_REF_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); - mHas.contextSwitches = - mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + if (!mHas.cpuCycles) { + // Fallback to cycles counter, reference cycles not available in many systems. + mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); + } mHas.instructions = mPc->monitor(PERF_COUNT_HW_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.instructions, true, true)); mHas.branchInstructions = mPc->monitor(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.branchInstructions, true, false)); mHas.branchMisses = mPc->monitor(PERF_COUNT_HW_BRANCH_MISSES, LinuxPerformanceCounters::Target(&mVal.branchMisses, true, false)); // mHas.branchMisses = false; + // SW events + mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + mHas.contextSwitches = + mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + mPc->start(); mPc->calibrate([] { auto before = ankerl::nanobench::Clock::now(); @@ -2789,7 +2803,7 @@ void StreamStateRestorer::restore() { Number::Number(int width, int precision, int64_t value) : mWidth(width) , mPrecision(precision) - , mValue(static_cast<double>(value)) {} + , mValue(d(value)) {} Number::Number(int width, int precision, double value) : mWidth(width) @@ -2823,7 +2837,7 @@ std::ostream& operator<<(std::ostream& os, Number const& n) { return n.write(os); } -MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) +MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept : mWidth(w) , mPrecision(prec) , mTitle(std::move(tit)) @@ -2884,14 +2898,14 @@ std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) { Config::Config() = default; Config::~Config() = default; Config& Config::operator=(Config const&) = default; -Config& Config::operator=(Config&&) noexcept = default; +Config& Config::operator=(Config&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Config::Config(Config const&) = default; Config::Config(Config&&) noexcept = default; // provide implementation here so it's only generated once Result::~Result() = default; Result& Result::operator=(Result const&) = default; -Result& Result::operator=(Result&&) noexcept = default; +Result& Result::operator=(Result&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Result::Result(Result const&) = default; Result::Result(Result&&) noexcept = default; @@ -2992,7 +3006,7 @@ double Result::medianAbsolutePercentError(Measure m) const { auto data = mNameToMeasurements[detail::u(m)]; // calculates MdAPE which is the median of percentage error - // see https://www.spiderfinancial.com/support/documentation/numxl/reference-manual/forecasting-performance/mdape + // see https://support.numxl.com/hc/en-us/articles/115001223503-MdAPE-Median-Absolute-Percentage-Error auto med = calcMedian(data); // transform the data to absolute error @@ -3106,7 +3120,7 @@ Bench::Bench() { } Bench::Bench(Bench&&) noexcept = default; -Bench& Bench::operator=(Bench&&) noexcept = default; +Bench& Bench::operator=(Bench&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Bench::Bench(Bench const&) = default; Bench& Bench::operator=(Bench const&) = default; Bench::~Bench() noexcept = default; @@ -3423,7 +3437,7 @@ BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure) sumMeasure += rm.second; } - auto n = static_cast<double>(rangeMeasure.size()); + auto n = detail::d(rangeMeasure.size()); auto mean = sumMeasure / n; mNormalizedRootMeanSquare = std::sqrt(err / n) / mean; } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index a9b197b190..0fb222f3b5 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -27,7 +27,7 @@ struct TestBlockAndIndex { std::byte a{0}; stream.write({&a, 1}); // Prevent compaction - stream >> block; + stream >> TX_WITH_WITNESS(block); blockHash = block.GetHash(); blockindex.phashBlock = &blockHash; diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 11f96b1005..e1a0bf138e 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -62,7 +62,7 @@ static void VerifyScriptBench(benchmark::Bench& bench) #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << txSpend; + stream << TX_WITH_WITNESS(txSpend); int csuccess = bitcoinconsensus_verify_script_with_amount( txCredit.vout[0].scriptPubKey.data(), txCredit.vout[0].scriptPubKey.size(), diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index fc83a4ad3a..995b4781fc 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -26,13 +26,13 @@ #include <scheduler.h> #include <script/sigcache.h> #include <util/chaintype.h> +#include <util/fs.h> #include <util/thread.h> #include <validation.h> #include <validationinterface.h> #include <cassert> #include <cstdint> -#include <filesystem> #include <functional> #include <iosfwd> #include <memory> @@ -50,8 +50,8 @@ int main(int argc, char* argv[]) << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; return 1; } - std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); - std::filesystem::create_directories(abs_datadir); + fs::path abs_datadir{fs::absolute(argv[1])}; + fs::create_directories(abs_datadir); // SETUP: Context diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 211b4740be..29306b2229 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -107,12 +107,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - for (size_t i = 0; i < pool->vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); + for (const auto& tx : pool->txns_randomized) { + uint64_t shortid = cmpctblock.GetShortID(tx->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = tx; have_txn[idit->second] = true; mempool_count++; } else { diff --git a/src/blockencodings.h b/src/blockencodings.h index afdfa426f1..fb0f734ff8 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -65,7 +65,7 @@ public: SERIALIZE_METHODS(BlockTransactions, obj) { - READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn)); + READWRITE(obj.blockhash, TX_WITH_WITNESS(Using<VectorFormatter<TransactionCompression>>(obj.txn))); } }; @@ -76,7 +76,7 @@ struct PrefilledTransaction { uint16_t index; CTransactionRef tx; - SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); } + SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), TX_WITH_WITNESS(Using<TransactionCompression>(obj.tx))); } }; typedef enum ReadStatus_t diff --git a/src/common/args.cpp b/src/common/args.cpp index ca04175696..1f25d13bee 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -28,7 +28,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <filesystem> #include <map> #include <optional> #include <stdexcept> @@ -720,6 +719,13 @@ fs::path ArgsManager::GetConfigFilePath() const return *Assert(m_config_path); } +void ArgsManager::SetConfigFilePath(fs::path path) +{ + LOCK(cs_args); + assert(!m_config_path); + m_config_path = path; +} + ChainType ArgsManager::GetChainType() const { std::variant<ChainType, std::string> arg = GetChainArg(); diff --git a/src/common/args.h b/src/common/args.h index ae3ed02bc7..1c5db718f4 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -180,6 +180,7 @@ protected: * Return config file path (read-only) */ fs::path GetConfigFilePath() const; + void SetConfigFilePath(fs::path); [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); /** diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index f949655909..b3fee1e8b1 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -16,8 +16,9 @@ bool CheckTransaction(const CTransaction& tx, TxValidationState& state) if (tx.vout.empty()) return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-empty"); // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) { return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-oversize"); + } // Check for negative or overflow output values (see CVE-2010-5139) CAmount nValueOut = 0; diff --git a/src/consensus/validation.h b/src/consensus/validation.h index d5bf08cd61..3fdc01e66b 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -54,6 +54,8 @@ enum class TxValidationResult { TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction + TX_RECONSIDERABLE, //!< fails some policy, but might be acceptable if submitted in a (different) package + TX_UNKNOWN, //!< transaction was not validated because package failed }; /** A "reason" why a block was invalid, suitable for determining whether the @@ -147,16 +149,16 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // weight = (stripped_size * 3) + total_size. static inline int32_t GetTransactionWeight(const CTransaction& tx) { - return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(tx)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(tx)); } static inline int64_t GetBlockWeight(const CBlock& block) { - return ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(block, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(block)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(block)); } static inline int64_t GetTransactionInputWeight(const CTxIn& txin) { // scriptWitness size is added here because witnesses and txins are split up in segwit serialization. - return ::GetSerializeSize(txin, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(txin, PROTOCOL_VERSION) + ::GetSerializeSize(txin.scriptWitness.stack, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(txin)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(txin)) + ::GetSerializeSize(txin.scriptWitness.stack); } /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ diff --git a/src/core_io.h b/src/core_io.h index 1f5ecbaea6..a104f240c1 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -51,9 +51,9 @@ bool ParseHashStr(const std::string& strHex, uint256& result); // core_write.cpp UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); +std::string EncodeHexTx(const CTransaction& tx, const bool without_witness = false); std::string SighashToStr(unsigned char sighash_type); void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false, const SigningProvider* provider = nullptr); -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, bool without_witness = false, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index dfabf3a0c2..bffb9e9042 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -142,9 +142,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with extended serialization support, and remember if the result successfully // consumes the entire input. if (try_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssData(tx_data); try { - ssData >> tx_extended; + ssData >> TX_WITH_WITNESS(tx_extended); if (ssData.empty()) ok_extended = true; } catch (const std::exception&) { // Fall through. @@ -160,9 +160,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with legacy serialization, and remember if the result successfully consumes the entire input. if (try_no_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + DataStream ssData(tx_data); try { - ssData >> tx_legacy; + ssData >> TX_NO_WITNESS(tx_legacy); if (ssData.empty()) ok_legacy = true; } catch (const std::exception&) { // Fall through. @@ -222,9 +222,9 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return false; std::vector<unsigned char> blockData(ParseHex(strHexBlk)); - CDataStream ssBlock(blockData, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssBlock(blockData); try { - ssBlock >> block; + ssBlock >> TX_WITH_WITNESS(block); } catch (const std::exception&) { return false; diff --git a/src/core_write.cpp b/src/core_write.cpp index 7cf019a42e..a2be17d1aa 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -140,10 +140,14 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco return str; } -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) +std::string EncodeHexTx(const CTransaction& tx, const bool without_witness) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); - ssTx << tx; + DataStream ssTx; + if (without_witness) { + ssTx << TX_NO_WITNESS(tx); + } else { + ssTx << TX_WITH_WITNESS(tx); + } return HexStr(ssTx); } @@ -168,7 +172,7 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, bool without_witness, const CTxUndo* txundo, TxVerbosity verbosity) { CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); @@ -177,7 +181,7 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry // Transaction version is actually unsigned in consensus checks, just signed in memory, // so cast to unsigned before giving it to the user. entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); - entry.pushKV("size", (int)::GetSerializeSize(tx, PROTOCOL_VERSION)); + entry.pushKV("size", tx.GetTotalSize()); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); entry.pushKV("locktime", (int64_t)tx.nLockTime); @@ -264,6 +268,6 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry } if (include_hex) { - entry.pushKV("hex", EncodeHexTx(tx, serialize_flags)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". + entry.pushKV("hex", EncodeHexTx(tx, without_witness)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". } } diff --git a/src/hash.h b/src/hash.h index d355b703ff..132e1a9fb3 100644 --- a/src/hash.h +++ b/src/hash.h @@ -146,23 +146,6 @@ public: } }; -class CHashWriter : public HashWriter -{ -private: - const int nVersion; - -public: - CHashWriter(int nVersionIn) : nVersion{nVersionIn} {} - - int GetVersion() const { return nVersion; } - - template<typename T> - CHashWriter& operator<<(const T& obj) { - ::Serialize(*this, obj); - return (*this); - } -}; - /** Reads data from an underlying stream, while hashing the read data. */ template <typename Source> class HashVerifier : public HashWriter diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 661406e122..c72dbf10bc 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -8,6 +8,7 @@ #include <crypto/hmac_sha256.h> #include <httpserver.h> #include <logging.h> +#include <netaddress.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <util/strencodings.h> diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index e16dd0f8bd..5d37fd0d8f 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -65,7 +65,7 @@ bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) vPos.reserve(block.data->vtx.size()); for (const auto& tx : block.data->vtx) { vPos.emplace_back(tx->GetHash(), pos); - pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); + pos.nTxOffset += ::GetSerializeSize(TX_WITH_WITNESS(*tx)); } return m_db->WriteTxs(vPos); } @@ -89,7 +89,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { return error("%s: fseek(...) failed", __func__); } - file >> tx; + file >> TX_WITH_WITNESS(tx); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } diff --git a/src/init.cpp b/src/init.cpp index 42331d37e8..d6dc62f707 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -343,11 +343,11 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - node.kernel.reset(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman.reset(); node.scheduler.reset(); + node.kernel.reset(); try { if (!fs::remove(GetPidFile(*node.args))) { @@ -455,9 +455,14 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-minimumchainwork=<hex>", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex(), signetChainParams->GetConsensus().nMinimumChainWork.GetHex()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)", - -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)", + MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-persistmempoolv1", + strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format " + "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)", + DEFAULT_PERSIST_V1_DAT), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " @@ -489,7 +494,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", strprintf("Accept connections from outside (default: %u if no -proxy, -connect or -maxconnections=0)", DEFAULT_LISTEN), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> automatic connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1751,11 +1756,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nMaxConnections = nMaxConnections; - connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); - connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); - connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; + connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index dea868f844..aeb3797392 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -244,7 +244,7 @@ public: // outputs in the same transaction) or have shared ancestry, the bump fees are calculated // independently, i.e. as if only one of them is spent. This may result in double-fee-bumping. This // caveat can be rectified per use of the sister-function CalculateCombinedBumpFee(…). - virtual std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Calculate the combined bump fee for an input set per the same strategy // as in CalculateIndividualBumpFees(…). @@ -252,7 +252,7 @@ public: // bump fees per outpoint, but a single bump fee for the shared ancestry. // The combined bump fee may be used to correct overestimation due to // shared ancestry by multiple UTXOs after coin selection. - virtual std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Get the node's package limits. //! Currently only returns the ancestor and descendant count limits, but could be enhanced to @@ -335,7 +335,7 @@ public: virtual void rpcRunLater(const std::string& name, std::function<void()> fn, int64_t seconds) = 0; //! Current RPC serialization flags. - virtual int rpcSerializationFlags() = 0; + virtual bool rpcSerializationWithoutWitness() = 0; //! Get settings value. virtual common::SettingsValue getSetting(const std::string& arg) = 0; @@ -395,6 +395,9 @@ public: //! Set mock time. virtual void setMockTime(int64_t time) = 0; + + //! Mock the scheduler to fast forward in time. + virtual void schedulerMockForward(std::chrono::seconds delta_seconds) = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 3f8df57124..aeb2612c07 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -204,8 +204,8 @@ public: //! Unset RPC timer interface. virtual void rpcUnsetTimerInterface(RPCTimerInterface* iface) = 0; - //! Get unspent outputs associated with a transaction. - virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; + //! Get unspent output associated with a transaction. + virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 4b896c11a3..6114236623 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -118,7 +118,7 @@ public: wallet::AddressPurpose* purpose) = 0; //! Get wallet address list. - virtual std::vector<WalletAddress> getAddresses() const = 0; + virtual std::vector<WalletAddress> getAddresses() = 0; //! Get receive requests. virtual std::vector<std::string> getAddressReceiveRequests() = 0; diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 1f175a5ccf..7c905ca4f4 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -83,7 +83,7 @@ private: const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block - LockPoints lockPoints; //!< Track the height and time at which tx was final + mutable LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these @@ -151,7 +151,7 @@ public: } // Update the LockPoints after a reorg - void UpdateLockPoints(const LockPoints& lp) + void UpdateLockPoints(const LockPoints& lp) const { lockPoints = lp; } @@ -172,8 +172,10 @@ public: Parents& GetMemPoolParents() const { return m_parents; } Children& GetMemPoolChildren() const { return m_children; } - mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable size_t idx_randomized; //!< Index in mempool's txns_randomized mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; +using CTxMemPoolEntryRef = CTxMemPoolEntry::CTxMemPoolEntryRef; + #endif // BITCOIN_KERNEL_MEMPOOL_ENTRY_H diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index 757be41b3c..d09fd2ba35 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -23,6 +23,8 @@ static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; /** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{false}; +/** Whether to fall back to legacy V1 serialization when writing mempool.dat */ +static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false}; @@ -56,6 +58,7 @@ struct MemPoolOptions { bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG}; bool require_standard{true}; bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF}; + bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index ff655c5ffa..0808d42452 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <logging.h> #include <primitives/transaction.h> +#include <random.h> #include <serialize.h> #include <streams.h> #include <sync.h> @@ -34,14 +35,14 @@ using fsbridge::FopenFn; namespace kernel { -static const uint64_t MEMPOOL_DUMP_VERSION = 1; +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); return false; @@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - if (version != MEMPOOL_DUMP_VERSION) { + std::vector<std::byte> xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { return false; } + file.SetXor(xor_key); uint64_t num; file >> num; while (num) { @@ -67,7 +74,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active CTransactionRef tx; int64_t nTime; int64_t nFeeDelta; - file >> tx; + file >> TX_WITH_WITNESS(tx); file >> nTime; file >> nFeeDelta; @@ -151,20 +158,25 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock auto mid = SteadyClock::now(); - try { - FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } - uint64_t version = MEMPOOL_DUMP_VERSION; + try { + const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; + std::vector<std::byte> xor_key(8); + if (!pool.m_persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { - file << *(i.tx); + file << TX_WITH_WITNESS(*(i.tx)); file << int64_t{count_seconds(i.m_time)}; file << int64_t{i.nFeeDelta}; mapDeltas.erase(i.tx->GetHash()); diff --git a/src/net.cpp b/src/net.cpp index 16075efdbb..a2f80cbcf7 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -417,21 +417,25 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? GetDefaultPort(pszDest) : m_params.GetDefaultPort()}; if (pszDest) { - const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; if (!resolved.empty()) { - const CService& rnd{resolved[GetRand(resolved.size())]}; - addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; - if (!addrConnect.IsValid()) { - LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); - return nullptr; - } - // It is possible that we already have a connection to the IP/port pszDest resolved to. - // In that case, drop the connection that was just created. - LOCK(m_nodes_mutex); - CNode* pnode = FindNode(static_cast<CService>(addrConnect)); - if (pnode) { - LogPrintf("Failed to open new connection, already connected\n"); - return nullptr; + Shuffle(resolved.begin(), resolved.end(), FastRandomContext()); + // If the connection is made by name, it can be the case that the name resolves to more than one address. + // We don't want to connect any more of them if we are already connected to one + for (const auto& r : resolved) { + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(r), NODE_NONE}; + if (!addrConnect.IsValid()) { + LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); + return nullptr; + } + // It is possible that we already have a connection to the IP/port pszDest resolved to. + // In that case, drop the connection that was just created. + LOCK(m_nodes_mutex); + CNode* pnode = FindNode(static_cast<CService>(addrConnect)); + if (pnode) { + LogPrintf("Not opening a connection to %s, already connected to %s\n", pszDest, addrConnect.ToStringAddrPort()); + return nullptr; + } } } } @@ -680,10 +684,8 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) } V1Transport::V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept : - m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) + m_magic_bytes{Params().MessageStart()}, m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) { - assert(std::size(Params().MessageStart()) == std::size(m_magic_bytes)); - m_magic_bytes = Params().MessageStart(); LOCK(m_recv_mutex); Reset(); } @@ -1730,7 +1732,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const CAddress& addr) { int nInbound = 0; - int nMaxInbound = nMaxConnections - m_max_outbound; AddWhitelistPermissionFlags(permission_flags, addr); if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::Implicit)) { @@ -1776,13 +1777,13 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. bool discouraged = m_banman && m_banman->IsDiscouraged(addr); - if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= nMaxInbound && discouraged) + if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= m_max_inbound && discouraged) { LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToStringAddrPort()); return; } - if (nInbound >= nMaxInbound) + if (nInbound >= m_max_inbound) { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection @@ -2733,7 +2734,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. - const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(m_max_automatic_connections - 1, 2)}; // Use BIP324 transport when both us and them have NODE_V2_P2P set. const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*strDest=*/nullptr, conn_type, use_v2transport); @@ -2754,7 +2755,7 @@ std::vector<CAddress> CConnman::GetCurrentBlockRelayOnlyConns() const return ret; } -std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const +std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo(bool include_connected) const { std::vector<AddedNodeInfo> ret; @@ -2789,6 +2790,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is an IP:port auto it = mapConnected.find(service); if (it != mapConnected.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = service; addedNode.fConnected = true; addedNode.fInbound = it->second; @@ -2797,6 +2801,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is a name auto it = mapConnectedByName.find(addr.m_added_node); if (it != mapConnectedByName.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = it->second.second; addedNode.fConnected = true; addedNode.fInbound = it->second.first; @@ -2815,21 +2822,19 @@ void CConnman::ThreadOpenAddedConnections() while (true) { CSemaphoreGrant grant(*semAddnode); - std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(/*include_connected=*/false); bool tried = false; for (const AddedNodeInfo& info : vInfo) { - if (!info.fConnected) { - if (!grant) { - // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting - // the addednodeinfo state might change. - break; - } - tried = true; - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; - grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); + if (!grant) { + // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting + // the addednodeinfo state might change. + break; } + tried = true; + CAddress addr(CService(), NODE_NONE); + OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); } // Retry every 60 seconds if a connection was attempted, otherwise two seconds if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) @@ -3208,18 +3213,17 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = std::make_unique<CSemaphore>(std::min(m_max_outbound, nMaxConnections)); + semOutbound = std::make_unique<CSemaphore>(std::min(m_max_automatic_outbound, m_max_automatic_connections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = std::make_unique<CSemaphore>(nMaxAddnode); + semAddnode = std::make_unique<CSemaphore>(m_max_addnode); } // // Start threads // assert(m_msgproc); - InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -3291,16 +3295,16 @@ void CConnman::Interrupt() condMsgProc.notify_all(); interruptNet(); - InterruptSocks5(true); + g_socks5_interrupt(); if (semOutbound) { - for (int i=0; i<m_max_outbound; i++) { + for (int i=0; i<m_max_automatic_outbound; i++) { semOutbound->post(); } } if (semAddnode) { - for (int i=0; i<nMaxAddnode; i++) { + for (int i=0; i<m_max_addnode; i++) { semAddnode->post(); } } @@ -3426,9 +3430,12 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const AddedNodeParams& add) { + const CService resolved(LookupNumeric(add.m_added_node, GetDefaultPort(add.m_added_node))); + const bool resolved_is_valid{resolved.IsValid()}; + LOCK(m_added_nodes_mutex); for (const auto& it : m_added_node_params) { - if (add.m_added_node == it.m_added_node) return false; + if (add.m_added_node == it.m_added_node || (resolved_is_valid && resolved == LookupNumeric(it.m_added_node, GetDefaultPort(it.m_added_node)))) return false; } m_added_node_params.push_back(add); @@ -372,7 +372,7 @@ public: class V1Transport final : public Transport { private: - MessageStartChars m_magic_bytes; + const MessageStartChars m_magic_bytes; const NodeId m_node_id; // Only for logging mutable Mutex m_recv_mutex; //!< Lock for receive state mutable CHash256 hasher GUARDED_BY(m_recv_mutex); @@ -1045,11 +1045,7 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - int nMaxConnections = 0; - int m_max_outbound_full_relay = 0; - int m_max_outbound_block_relay = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; + int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; BanMan* m_banman = nullptr; @@ -1076,13 +1072,12 @@ public: AssertLockNotHeld(m_total_bytes_sent_mutex); nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); - m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; + m_max_automatic_connections = connOptions.m_max_automatic_connections; + m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); + m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); + m_max_automatic_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + m_max_feeler; + m_max_inbound = std::max(0, m_max_automatic_connections - m_max_automatic_outbound); m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; - m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; m_client_interface = connOptions.uiInterface; m_banman = connOptions.m_banman; m_msgproc = connOptions.m_msgproc; @@ -1189,7 +1184,7 @@ public: bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); - std::vector<AddedNodeInfo> GetAddedNodeInfo() const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + std::vector<AddedNodeInfo> GetAddedNodeInfo(bool include_connected) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); /** * Attempts to open a connection. Currently only used from tests. @@ -1466,7 +1461,18 @@ private: std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; - int nMaxConnections; + + /** + * Maximum number of automatic connections permitted, excluding manual + * connections but including inbounds. May be changed by the user and is + * potentially limited by the operating system (number of file descriptors). + */ + int m_max_automatic_connections; + + /* + * Maximum number of peers by connection type. Might vary from defaults + * based on -maxconnections init value. + */ // How many full-relay (tx, block, addr) outbound peers we want int m_max_outbound_full_relay; @@ -1475,9 +1481,11 @@ private: // We do not relay tx or addr messages with these peers int m_max_outbound_block_relay; - int nMaxAddnode; - int nMaxFeeler; - int m_max_outbound; + int m_max_addnode{MAX_ADDNODE_CONNECTIONS}; + int m_max_feeler{MAX_FEELER_CONNECTIONS}; + int m_max_automatic_outbound; + int m_max_inbound; + bool m_use_addrman_outgoing; CClientUIInterface* m_client_interface; NetEventsInterface* m_msgproc; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 611bd47e7f..1556dc7e67 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1820,6 +1820,8 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: case TxValidationResult::TX_NO_MEMPOOL: + case TxValidationResult::TX_RECONSIDERABLE: + case TxValidationResult::TX_UNKNOWN: break; } return false; @@ -2306,9 +2308,9 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } if (pblock) { if (inv.IsMsgBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_NO_WITNESS(*pblock))); } else if (inv.IsMsgWitnessBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock))); } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; @@ -2329,7 +2331,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::TX, TX_NO_WITNESS(*pblock->vtx[pair.first]))); } // else // no response @@ -2346,7 +2348,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock))); } } } @@ -2416,8 +2418,8 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv)); if (tx) { // WTX and WITNESS_TX imply we serialize with witness - int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + const auto maybe_with_witness = (inv.IsMsgTx() ? TX_NO_WITNESS : TX_WITH_WITNESS); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::TX, maybe_with_witness(*tx))); m_mempool.RemoveUnbroadcastTx(tx->GetHash()); } else { vNotFound.push_back(inv); @@ -4117,7 +4119,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); // Just respond with an empty headers message, to tell the peer to // go away but not treat us as unresponsive. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlock>())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlockHeader>())); return; } @@ -4167,7 +4169,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : m_chainman.ActiveChain().Tip(); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders))); return; } @@ -4184,7 +4186,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (m_chainman.IsInitialBlockDownload()) return; CTransactionRef ptx; - vRecv >> ptx; + vRecv >> TX_WITH_WITNESS(ptx); const CTransaction& tx = *ptx; const uint256& txid = ptx->GetHash(); @@ -4685,7 +4687,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); - vRecv >> *pblock; + vRecv >> TX_WITH_WITNESS(*pblock); LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom.GetId()); @@ -5696,7 +5698,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders))); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; diff --git a/src/netbase.cpp b/src/netbase.cpp index 5a609a872e..9fbd9f7dea 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -30,7 +30,7 @@ bool fNameLookup = DEFAULT_NAME_LOOKUP; // Need ample time for negotiation for very slow proxies such as Tor std::chrono::milliseconds g_socks5_recv_timeout = 20s; -static std::atomic<bool> interruptSocks5Recv(false); +CThreadInterrupt g_socks5_interrupt; ReachableNets g_reachable_nets; @@ -271,7 +271,7 @@ enum class IntrRecvError { * IntrRecvError::OK only if all of the specified number of bytes were * read. * - * @see This function can be interrupted by calling InterruptSocks5(bool). + * @see This function can be interrupted by calling g_socks5_interrupt(). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) @@ -299,8 +299,9 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::m return IntrRecvError::NetworkError; } } - if (interruptSocks5Recv) + if (g_socks5_interrupt) { return IntrRecvError::Interrupted; + } curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; @@ -333,103 +334,93 @@ static std::string Socks5ErrorString(uint8_t err) bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* auth, const Sock& sock) { - IntrRecvError recvr; - LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); - if (strDest.size() > 255) { - return error("Hostname too long"); - } - // Construct the version identifier/method selection message - std::vector<uint8_t> vSocks5Init; - vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol - if (auth) { - vSocks5Init.push_back(0x02); // 2 method identifiers follow... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - vSocks5Init.push_back(SOCKS5Method::USER_PASS); - } else { - vSocks5Init.push_back(0x01); // 1 method identifier follows... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - } - ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5Init.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet1[2]; - if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); - return false; - } - if (pchRet1[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to initialize"); - } - if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { - // Perform username/password authentication (as described in RFC1929) - std::vector<uint8_t> vAuth; - vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation - if (auth->username.size() > 255 || auth->password.size() > 255) - return error("Proxy username or password too long"); - vAuth.push_back(auth->username.size()); - vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); - vAuth.push_back(auth->password.size()); - vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vAuth.size()) { - return error("Error sending authentication to proxy"); - } - LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); - uint8_t pchRetA[2]; - if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading proxy authentication response"); + try { + IntrRecvError recvr; + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); + if (strDest.size() > 255) { + return error("Hostname too long"); } - if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - return error("Proxy authentication unsuccessful"); + // Construct the version identifier/method selection message + std::vector<uint8_t> vSocks5Init; + vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol + if (auth) { + vSocks5Init.push_back(0x02); // 2 method identifiers follow... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); + vSocks5Init.push_back(SOCKS5Method::USER_PASS); + } else { + vSocks5Init.push_back(0x01); // 1 method identifier follows... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { - // Perform no authentication - } else { - return error("Proxy requested wrong authentication method %02x", pchRet1[1]); - } - std::vector<uint8_t> vSocks5; - vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version - vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT - vSocks5.push_back(0x00); // RSV Reserved must be 0 - vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME - vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function - vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); - vSocks5.push_back((port >> 8) & 0xFF); - vSocks5.push_back((port >> 0) & 0xFF); - ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { - if (recvr == IntrRecvError::Timeout) { - /* If a timeout happens here, this effectively means we timed out while connecting - * to the remote node. This is very common for Tor, so do not print an - * error message. */ + sock.SendComplete(vSocks5Init, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet1[2]; + if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; + } + if (pchRet1[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to initialize"); + } + if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { + // Perform username/password authentication (as described in RFC1929) + std::vector<uint8_t> vAuth; + vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation + if (auth->username.size() > 255 || auth->password.size() > 255) + return error("Proxy username or password too long"); + vAuth.push_back(auth->username.size()); + vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); + vAuth.push_back(auth->password.size()); + vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); + sock.SendComplete(vAuth, g_socks5_recv_timeout, g_socks5_interrupt); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + uint8_t pchRetA[2]; + if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading proxy authentication response"); + } + if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { + return error("Proxy authentication unsuccessful"); + } + } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { + // Perform no authentication } else { - return error("Error while reading proxy response"); + return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } - } - if (pchRet2[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to accept request"); - } - if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { - // Failures to connect to a peer that are not proxy errors - LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); - return false; - } - if (pchRet2[2] != 0x00) { // Reserved field must be 0 - return error("Error: malformed proxy response"); - } - uint8_t pchRet3[256]; - switch (pchRet2[3]) - { + std::vector<uint8_t> vSocks5; + vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version + vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT + vSocks5.push_back(0x00); // RSV Reserved must be 0 + vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME + vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function + vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); + vSocks5.push_back((port >> 8) & 0xFF); + vSocks5.push_back((port >> 0) & 0xFF); + sock.SendComplete(vSocks5, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet2[4]; + if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { + if (recvr == IntrRecvError::Timeout) { + /* If a timeout happens here, this effectively means we timed out while connecting + * to the remote node. This is very common for Tor, so do not print an + * error message. */ + return false; + } else { + return error("Error while reading proxy response"); + } + } + if (pchRet2[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to accept request"); + } + if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { + // Failures to connect to a peer that are not proxy errors + LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); + return false; + } + if (pchRet2[2] != 0x00) { // Reserved field must be 0 + return error("Error: malformed proxy response"); + } + uint8_t pchRet3[256]; + switch (pchRet2[3]) { case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, g_socks5_recv_timeout, sock); break; case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, g_socks5_recv_timeout, sock); break; - case SOCKS5Atyp::DOMAINNAME: - { + case SOCKS5Atyp::DOMAINNAME: { recvr = InterruptibleRecv(pchRet3, 1, g_socks5_recv_timeout, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); @@ -439,15 +430,18 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a break; } default: return error("Error: malformed proxy response"); + } + if (recvr != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); + return true; + } catch (const std::runtime_error& e) { + return error("Error during SOCKS5 proxy handshake: %s", e.what()); } - if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); - return true; } std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) @@ -681,11 +675,6 @@ CSubNet LookupSubNet(const std::string& subnet_str) return subnet; } -void InterruptSocks5(bool interrupt) -{ - interruptSocks5Recv = interrupt; -} - bool IsBadPort(uint16_t port) { /* Don't forget to update doc/p2p-bad-ports.md if you change this list. */ diff --git a/src/netbase.h b/src/netbase.h index d51f63fd81..8523f59b4d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -13,6 +13,7 @@ #include <netaddress.h> #include <serialize.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <functional> #include <memory> @@ -274,7 +275,10 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -void InterruptSocks5(bool interrupt); +/** + * Interrupt SOCKS5 reads or writes. + */ +extern CThreadInterrupt g_socks5_interrupt; /** * Connect to a specified destination service through an already connected diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 058e524745..1108cc676b 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -956,7 +956,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const } // Write index header - unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); + unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block)); fileout << GetParams().MessageStart() << nSize; // Write block @@ -965,7 +965,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const return error("WriteBlockToDisk: ftell failed"); } pos.nPos = (unsigned int)fileOutPos; - fileout << block; + fileout << TX_WITH_WITNESS(block); return true; } @@ -1023,7 +1023,7 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) cons // Read block try { - filein >> block; + filein >> TX_WITH_WITNESS(block); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } @@ -1092,7 +1092,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { - unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); + unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block)); FlatFilePos blockPos; const auto position_known {dbp != nullptr}; if (position_known) { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f6dbe4f008..f4ecfeb9d5 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -328,10 +328,12 @@ public: std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); } void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } - bool getUnspentOutput(const COutPoint& output, Coin& coin) override + std::optional<Coin> getUnspentOutput(const COutPoint& output) override { LOCK(::cs_main); - return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); + Coin coin; + if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin; + return {}; } TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override { @@ -648,8 +650,9 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - auto it = m_node.mempool->GetIter(txid); - return it && (*it)->GetCountWithDescendants() > 1; + const auto entry{m_node.mempool->GetEntry(Txid::FromUint256(txid))}; + if (entry == nullptr) return false; + return entry->GetCountWithDescendants() > 1; } bool broadcastTransaction(const CTransactionRef& tx, const CAmount& max_tx_fee, @@ -669,7 +672,7 @@ public: m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants, ancestorsize, ancestorfees); } - std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { std::map<COutPoint, CAmount> bump_fees; @@ -681,7 +684,7 @@ public: return MiniMiner(*m_node.mempool, outpoints).CalculateBumpFees(target_feerate); } - std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { return 0; @@ -702,9 +705,9 @@ public: if (!m_node.mempool) return true; LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); - const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); - return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); + std::string err_string; + return m_node.mempool->CheckPackageLimits({tx}, entry.GetTxSize(), err_string); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { @@ -772,7 +775,7 @@ public: { RPCRunLater(name, std::move(fn), seconds); } - int rpcSerializationFlags() override { return RPCSerializationFlags(); } + bool rpcSerializationWithoutWitness() override { return RPCSerializationWithoutWitness(); } common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); @@ -806,7 +809,7 @@ public: { if (!m_node.mempool) return; LOCK2(::cs_main, m_node.mempool->cs); - for (const CTxMemPoolEntry& entry : m_node.mempool->mapTx) { + for (const CTxMemPoolEntry& entry : m_node.mempool->entryAll()) { notifications.transactionAddedToMempool(entry.GetSharedTx()); } } diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index f63d9875fc..ac26600919 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -93,6 +93,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); + mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); + ApplyArgsManOptions(argsman, mempool_opts.limits); return {}; diff --git a/src/node/miner.cpp b/src/node/miner.cpp index caa2991819..ce5452d1f9 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -188,7 +188,7 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) { for (CTxMemPool::setEntries::iterator iit = testSet.begin(); iit != testSet.end(); ) { // Only test txs not already in the block - if (inBlock.count(*iit)) { + if (inBlock.count((*iit)->GetSharedTx()->GetHash())) { testSet.erase(iit++); } else { iit++; @@ -229,7 +229,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) ++nBlockTx; nBlockSigOpsCost += iter->GetSigOpCost(); nFees += iter->GetFee(); - inBlock.insert(iter); + inBlock.insert(iter->GetSharedTx()->GetHash()); bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { @@ -298,7 +298,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // because some of their txs are already in the block indexed_modified_transaction_set mapModifiedTx; // Keep track of entries that failed inclusion, to avoid duplicate work - CTxMemPool::setEntries failedTx; + std::set<Txid> failedTx; CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; @@ -326,7 +326,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (mi != mempool.mapTx.get<ancestor_score>().end()) { auto it = mempool.mapTx.project<0>(mi); assert(it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) { + if (mapModifiedTx.count(it) || inBlock.count(it->GetSharedTx()->GetHash()) || failedTx.count(it->GetSharedTx()->GetHash())) { ++mi; continue; } @@ -360,7 +360,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't // contain anything that is inBlock. - assert(!inBlock.count(iter)); + assert(!inBlock.count(iter->GetSharedTx()->GetHash())); uint64_t packageSize = iter->GetSizeWithAncestors(); CAmount packageFees = iter->GetModFeesWithAncestors(); @@ -382,7 +382,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // we must erase failed entries so that we can consider the // next best entry on the next loop iteration mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } ++nConsecutiveFailed; @@ -404,7 +404,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (!TestPackageTransactions(ancestors)) { if (fUsingModified) { mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } continue; } diff --git a/src/node/miner.h b/src/node/miner.h index 4173521585..06a917228d 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -142,7 +142,7 @@ private: uint64_t nBlockTx; uint64_t nBlockSigOpsCost; CAmount nFees; - CTxMemPool::setEntries inBlock; + std::unordered_set<Txid, SaltedTxidHasher> inBlock; // Chain context for the block int nHeight; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp index 3d24a3f58e..58422c4439 100644 --- a/src/node/mini_miner.cpp +++ b/src/node/mini_miner.cpp @@ -78,11 +78,11 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou for (const auto& txiter : cluster) { if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), - MiniMinerMempoolEntry{/*fee_self=*/txiter->GetModifiedFee(), - /*fee_ancestor=*/txiter->GetModFeesWithAncestors(), + MiniMinerMempoolEntry{/*tx_in=*/txiter->GetSharedTx(), /*vsize_self=*/txiter->GetTxSize(), /*vsize_ancestor=*/txiter->GetSizeWithAncestors(), - /*tx_in=*/txiter->GetSharedTx()}); + /*fee_self=*/txiter->GetModifiedFee(), + /*fee_ancestor=*/txiter->GetModFeesWithAncestors()}); m_entries.push_back(mapiter); } else { auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); @@ -154,7 +154,7 @@ MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, m_ready_to_calculate = false; return; } - std::vector<MockEntryMap::iterator> cached_descendants; + std::vector<MockEntryMap::iterator> descendants; for (const auto& desc_txid : desc_txids) { auto desc_it{m_entries_by_txid.find(desc_txid)}; // Descendants should only include transactions with corresponding entries. @@ -162,10 +162,10 @@ MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, m_ready_to_calculate = false; return; } else { - cached_descendants.emplace_back(desc_it); + descendants.emplace_back(desc_it); } } - m_descendant_set_by_txid.emplace(txid, cached_descendants); + m_descendant_set_by_txid.emplace(txid, descendants); } Assume(m_to_be_replaced.empty()); Assume(m_requested_outpoints_by_txid.empty()); diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h index 8f86709ae4..de62c0af75 100644 --- a/src/node/mini_miner.h +++ b/src/node/mini_miner.h @@ -34,11 +34,11 @@ class MiniMinerMempoolEntry // methods can be called without holding that lock. public: - explicit MiniMinerMempoolEntry(CAmount fee_self, - CAmount fee_ancestor, + explicit MiniMinerMempoolEntry(const CTransactionRef& tx_in, int64_t vsize_self, int64_t vsize_ancestor, - const CTransactionRef& tx_in): + CAmount fee_self, + CAmount fee_ancestor): tx{tx_in}, vsize_individual{vsize_self}, vsize_with_ancestors{vsize_ancestor}, @@ -137,10 +137,10 @@ public: */ MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); - /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually, - * presumably because these transactions are not in the mempool (yet). It is assumed that all - * entries are unique and their values are correct, otherwise results computed by MiniMiner may - * be incorrect. Callers should check IsReadyToCalculate() after construction. + /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually. + * It is assumed that all entries are unique and their values are correct, otherwise results + * computed by MiniMiner may be incorrect. Callers should check IsReadyToCalculate() after + * construction. * @param[in] descendant_caches A map from each transaction to the set of txids of this * transaction's descendant set, including itself. Each tx in * manual_entries must have a corresponding entry in this map, and diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 9557594622..654c4cf0ce 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1044,8 +1044,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed() std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge() { - auto file_time = std::filesystem::last_write_time(m_estimation_filepath); - auto now = std::filesystem::file_time_type::clock::now(); + auto file_time{fs::last_write_time(m_estimation_filepath)}; + auto now{fs::file_time_type::clock::now()}; return std::chrono::duration_cast<std::chrono::hours>(now - file_time); } diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index d032b74008..76ab2b1a96 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -12,6 +12,7 @@ #include <tinyformat.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/rbf.h> @@ -35,7 +36,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. - const CTxMemPoolEntry entry{*pool.mapTx.find(tx.GetHash())}; + const auto& entry{*Assert(pool.GetEntry(tx.GetHash()))}; auto ancestors{pool.AssumeCalculateMemPoolAncestors(__func__, entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 3d21708820..27da161e1e 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -10,7 +10,7 @@ uint256 CBlockHeader::GetHash() const { - return (CHashWriter{PROTOCOL_VERSION} << *this).GetHash(); + return (HashWriter{} << *this).GetHash(); } std::string CBlock::ToString() const diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 1ad8345fcb..6650277c2b 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -68,12 +68,12 @@ CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), Txid CMutableTransaction::GetHash() const { - return Txid::FromUint256((CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash()); + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); } Txid CTransaction::ComputeHash() const { - return Txid::FromUint256((CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash()); + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); } Wtxid CTransaction::ComputeWitnessHash() const @@ -82,7 +82,7 @@ Wtxid CTransaction::ComputeWitnessHash() const return Wtxid::FromUint256(hash.ToUint256()); } - return Wtxid::FromUint256((CHashWriter{0} << *this).GetHash()); + return Wtxid::FromUint256((HashWriter{} << TX_WITH_WITNESS(*this)).GetHash()); } CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} @@ -102,7 +102,7 @@ CAmount CTransaction::GetValueOut() const unsigned int CTransaction::GetTotalSize() const { - return ::GetSerializeSize(*this, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_WITH_WITNESS(*this)); } std::string CTransaction::ToString() const diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 89deb9de4d..ad190f7d7f 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -24,14 +24,6 @@ #include <utility> #include <vector> -/** - * A flag that is ORed into the protocol version to designate that a transaction - * should be (un)serialized without witness data. - * Make sure that this does not collide with any of the values in `version.h` - * or with `ADDRV2_FORMAT`. - */ -static const int SERIALIZE_TRANSACTION_NO_WITNESS = 0x40000000; - /** An outpoint - a combination of a transaction hash and an index n into its vout */ class COutPoint { @@ -197,6 +189,13 @@ public: struct CMutableTransaction; +struct TransactionSerParams { + const bool allow_witness; + SER_PARAMS_OPFUNC +}; +static constexpr TransactionSerParams TX_WITH_WITNESS{.allow_witness = true}; +static constexpr TransactionSerParams TX_NO_WITNESS{.allow_witness = false}; + /** * Basic transaction serialization format: * - int32_t nVersion @@ -215,8 +214,9 @@ struct CMutableTransaction; * - uint32_t nLockTime */ template<typename Stream, typename TxType> -inline void UnserializeTransaction(TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void UnserializeTransaction(TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s >> tx.nVersion; unsigned char flags = 0; @@ -254,8 +254,9 @@ inline void UnserializeTransaction(TxType& tx, Stream& s) { } template<typename Stream, typename TxType> -inline void SerializeTransaction(const TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void SerializeTransaction(const TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s << tx.nVersion; unsigned char flags = 0; @@ -323,13 +324,15 @@ public: template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } /** This deserializing constructor is provided instead of an Unserialize method. * Unserialize is not possible, since it would require overwriting const fields. */ template <typename Stream> - CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} + CTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) : CTransaction(CMutableTransaction(deserialize, params, s)) {} + template <typename Stream> + CTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) : CTransaction(CMutableTransaction(deserialize, s)) {} bool IsNull() const { return vin.empty() && vout.empty(); @@ -389,17 +392,21 @@ struct CMutableTransaction template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } - template <typename Stream> inline void Unserialize(Stream& s) { - UnserializeTransaction(*this, s); + UnserializeTransaction(*this, s, s.GetParams()); + } + + template <typename Stream> + CMutableTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) { + UnserializeTransaction(*this, s, params); } template <typename Stream> - CMutableTransaction(deserialize_type, Stream& s) { + CMutableTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) { Unserialize(s); } diff --git a/src/protocol.cpp b/src/protocol.cpp index f956728af2..27a0a2ffc1 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -91,9 +91,8 @@ const static std::vector<std::string> g_all_net_message_types{ }; CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) + : pchMessageStart{pchMessageStartIn} { - pchMessageStart = pchMessageStartIn; - // Copy the command name size_t i = 0; for (; i < COMMAND_SIZE && pszCommand[i] != 0; ++i) pchCommand[i] = pszCommand[i]; diff --git a/src/psbt.h b/src/psbt.h index 5b4daafed5..1fd186f6a5 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -95,7 +95,7 @@ void SerializeToVector(Stream& s, const X&... args) // Takes a stream and multiple arguments and unserializes them first as a vector then each object individually in the order provided in the arguments template<typename Stream, typename... X> -void UnserializeFromVector(Stream& s, X&... args) +void UnserializeFromVector(Stream& s, X&&... args) { size_t expected_size = ReadCompactSize(s); size_t remaining_before = s.size(); @@ -226,8 +226,7 @@ struct PSBTInput // Write the utxo if (non_witness_utxo) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_NON_WITNESS_UTXO)); - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, non_witness_utxo); + SerializeToVector(s, TX_NO_WITNESS(non_witness_utxo)); } if (!witness_utxo.IsNull()) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_WITNESS_UTXO)); @@ -394,8 +393,7 @@ struct PSBTInput throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); } // Set the stream to unserialize with witness since this is always a valid network transaction - OverrideStream<Stream> os{&s, s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, non_witness_utxo); + UnserializeFromVector(s, TX_WITH_WITNESS(non_witness_utxo)); break; } case PSBT_IN_WITNESS_UTXO: @@ -984,8 +982,7 @@ struct PartiallySignedTransaction SerializeToVector(s, CompactSizeWriter(PSBT_GLOBAL_UNSIGNED_TX)); // Write serialized tx to a stream - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, *tx); + SerializeToVector(s, TX_NO_WITNESS(*tx)); // Write xpubs for (const auto& xpub_pair : m_xpubs) { @@ -1075,8 +1072,7 @@ struct PartiallySignedTransaction } CMutableTransaction mtx; // Set the stream to serialize with non-witness since this should always be non-witness - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, mtx); + UnserializeFromVector(s, TX_NO_WITNESS(mtx)); tx = std::move(mtx); // Make sure that all scriptSigs and scriptWitnesses are empty for (const CTxIn& txin : tx->vin) { diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 51f6f44923..a916e4ead6 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -360,12 +360,10 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall { COutPoint prevout = txin.prevout; - Coin prev; - if(node.getUnspentOutput(prevout, prev)) - { + if (auto prev{node.getUnspentOutput(prevout)}) { { strHTML += "<li>"; - const CTxOut &vout = prev.out; + const CTxOut& vout = prev->out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index a45579fa0d..445495897d 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -260,8 +260,8 @@ void WalletModel::sendCoins(WalletModelTransaction& transaction) auto& newTx = transaction.getWtx(); wallet().commitTransaction(newTx, /*value_map=*/{}, std::move(vOrderForm)); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << *newTx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(*newTx); transaction_array.append((const char*)ssTx.data(), ssTx.size()); } diff --git a/src/random.cpp b/src/random.cpp index 9bd8ff9f1a..ce4266a567 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -38,6 +38,9 @@ #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif +#if defined(HAVE_STRONG_GETAUXVAL) && defined(__aarch64__) +#include <sys/auxv.h> +#endif [[noreturn]] static void RandFailure() { @@ -173,6 +176,62 @@ static uint64_t GetRdSeed() noexcept #endif } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + +static bool g_rndr_supported = false; + +static void InitHardwareRand() +{ + if (getauxval(AT_HWCAP2) & HWCAP2_RNG) { + g_rndr_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rndr_supported) { + LogPrintf("Using RNDR and RNDRRS as additional entropy sources\n"); + } +} + +/** Read 64 bits of entropy using rndr. + * + * Must only be called when RNDR is supported. + */ +static uint64_t GetRNDR() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDR--Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_0; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + +/** Read 64 bits of entropy using rndrrs. + * + * Must only be called when RNDRRS is supported. + */ +static uint64_t GetRNDRRS() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDRRS--Reseeded-Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_1; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -191,6 +250,12 @@ static void SeedHardwareFast(CSHA512& hasher) noexcept { hasher.Write((const unsigned char*)&out, sizeof(out)); return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + uint64_t out = GetRNDR(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif } @@ -216,6 +281,14 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { } return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRNDRRS(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } #endif } diff --git a/src/rest.cpp b/src/rest.cpp index e094039366..f86c47ee6b 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -26,6 +26,7 @@ #include <txmempool.h> #include <util/any.h> #include <util/check.h> +#include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -316,8 +317,8 @@ static bool rest_block(const std::any& context, switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string binaryBlock = ssBlock.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryBlock); @@ -325,8 +326,8 @@ static bool rest_block(const std::any& context, } case RESTResponseFormat::HEX: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); @@ -722,8 +723,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << RPCTxSerParams(tx); std::string binaryTx = ssTx.str(); req->WriteHeader("Content-Type", "application/octet-stream"); @@ -732,8 +733,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string } case RESTResponseFormat::HEX: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << RPCTxSerParams(tx); std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b84747a3f..9a29b73bee 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -166,8 +166,8 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn { UniValue result = blockheaderToJSON(tip, blockindex); - result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); - result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); + result.pushKV("strippedsize", (int)::GetSerializeSize(TX_NO_WITNESS(block))); + result.pushKV("size", (int)::GetSerializeSize(TX_WITH_WITNESS(block))); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); @@ -189,7 +189,7 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn // coinbase transaction (i.e. i == 0) doesn't have undo data const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, /*without_witness=*/RPCSerializationWithoutWitness(), txundo, verbosity); txs.push_back(objTx); } break; @@ -740,8 +740,8 @@ static RPCHelpMan getblock() if (verbosity <= 0) { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string strHex = HexStr(ssBlock); return strHex; } diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 23cef42d30..bf60eae433 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -21,6 +21,7 @@ #include <univalue.h> #include <util/fs.h> #include <util/moneystr.h> +#include <util/strencodings.h> #include <util/time.h> #include <utility> @@ -289,7 +290,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("descendantsize", e.GetSizeWithDescendants()); info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", e.GetTx().GetWitnessHash().ToString()); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -315,9 +316,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { + for (const CTxMemPoolEntry& child : e.GetMemPoolChildrenConst()) { spent.push_back(child.GetTx().GetHash().ToString()); } @@ -344,14 +343,13 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo } LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); + for (const CTxMemPoolEntry& e : pool.entryAll()) { UniValue info(UniValue::VOBJ); entryToJSON(pool, info, e); // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). // UniValue::pushKVEnd is used instead which currently is O(1). - o.pushKVEnd(hash.ToString(), info); + o.pushKVEnd(e.GetTx().GetHash().ToString(), info); } return o; } else { @@ -460,12 +458,12 @@ static RPCHelpMan getmempoolancestors() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *it, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; + auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; if (!fVerbose) { UniValue o(UniValue::VARR); @@ -521,15 +519,15 @@ static RPCHelpMan getmempooldescendants() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto it{mempool.GetIter(hash)}; + if (!it) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); + mempool.CalculateDescendants(*it, setDescendants); // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); + setDescendants.erase(*it); if (!fVerbose) { UniValue o(UniValue::VARR); @@ -573,14 +571,13 @@ static RPCHelpMan getmempoolentry() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); + entryToJSON(mempool, info, *entry); return info; }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 76170c3201..1bdc1e1029 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -390,8 +390,8 @@ static RPCHelpMan generateblock() UniValue obj(UniValue::VOBJ); obj.pushKV("hash", block_out->GetHash().GetHex()); if (!process_new_block) { - CDataStream block_ser{SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()}; - block_ser << *block_out; + DataStream block_ser; + block_ser << RPCTxSerParams(*block_out); obj.pushKV("hex", HexStr(block_ser)); } return obj; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1a9052e008..c631132df2 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -16,6 +16,7 @@ #include <netbase.h> #include <node/context.h> #include <policy/settings.h> +#include <protocol.h> #include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/server_util.h> @@ -98,6 +99,18 @@ static RPCHelpMan ping() }; } +/** Returns, given services flags, a list of humanly readable (known) network services */ +static UniValue GetServicesNames(ServiceFlags services) +{ + UniValue servicesNames(UniValue::VARR); + + for (const auto& flag : serviceFlagsToStr(services)) { + servicesNames.push_back(flag); + } + + return servicesNames; +} + static RPCHelpMan getpeerinfo() { return RPCHelpMan{ @@ -487,7 +500,7 @@ static RPCHelpMan getaddednodeinfo() NodeContext& node = EnsureAnyNodeContext(request.context); const CConnman& connman = EnsureConnman(node); - std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(/*include_connected=*/true); if (!request.params[0].isNull()) { bool found = false; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 6b3662996c..b085828215 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -91,6 +91,9 @@ static RPCHelpMan mockscheduler() const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds}); SyncWithValidationInterfaceQueue(); + for (const auto& chain_client : node_context.chain_clients) { + chain_client->schedulerMockForward(std::chrono::seconds(delta_seconds)); + } return UniValue::VNULL; }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 16705b3ce2..397ece08c0 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -62,7 +62,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationWithoutWitness(), txundo, verbosity); if (!hashBlock.IsNull()) { LOCK(cs_main); @@ -383,7 +383,7 @@ static RPCHelpMan getrawtransaction() } if (verbosity <= 0) { - return EncodeHexTx(*tx, RPCSerializationFlags()); + return EncodeHexTx(*tx, /*without_witness=*/RPCSerializationWithoutWitness()); } UniValue result(UniValue::VOBJ); @@ -1541,7 +1541,7 @@ static RPCHelpMan finalizepsbt() std::string result_str; if (complete && extract) { - ssTx << mtx; + ssTx << TX_WITH_WITNESS(mtx); result_str = HexStr(ssTx); result.pushKV("hex", result_str); } else { @@ -1994,8 +1994,8 @@ RPCHelpMan descriptorprocesspsbt() CMutableTransaction mtx; PartiallySignedTransaction psbtx_copy = psbtx; CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx_copy, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } return result; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index ab410ffc05..7adc28f416 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -595,12 +595,9 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } -int RPCSerializationFlags() +bool RPCSerializationWithoutWitness() { - int flag = 0; - if (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) - flag |= SERIALIZE_TRANSACTION_NO_WITNESS; - return flag; + return (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0); } CRPCTable tableRPC; diff --git a/src/rpc/server.h b/src/rpc/server.h index 24658ddb8b..9a49d82570 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -183,7 +183,14 @@ void InterruptRPC(); void StopRPC(); std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); -// Retrieves any serialization flags requested in command line argument -int RPCSerializationFlags(); +// Drop witness when serializing for RPC? +bool RPCSerializationWithoutWitness(); + +template<typename T> +auto RPCTxSerParams(T&& t) +{ + if (RPCSerializationWithoutWitness()) return TX_NO_WITNESS(t); + return TX_WITH_WITNESS(t); +} #endif // BITCOIN_RPC_SERVER_H diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 5068d016b6..cf48ee11e7 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1304,17 +1304,6 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl return ret; } -UniValue GetServicesNames(ServiceFlags services) -{ - UniValue servicesNames(UniValue::VARR); - - for (const auto& flag : serviceFlagsToStr(services)) { - servicesNames.push_back(flag); - } - - return servicesNames; -} - /** Convert a vector of bilingual strings to a UniValue::VARR containing their original untranslated values. */ [[nodiscard]] static UniValue BilingualStringsToUniValue(const std::vector<bilingual_str>& bilingual_strings) { diff --git a/src/rpc/util.h b/src/rpc/util.h index ed14986c49..e2d5ed333c 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -9,7 +9,6 @@ #include <consensus/amount.h> #include <node/transaction.h> #include <outputtype.h> -#include <protocol.h> #include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -60,7 +59,6 @@ extern const std::string UNIX_EPOCH_TIME; extern const std::string EXAMPLE_ADDRESS[2]; class FillableSigningProvider; -class CPubKey; class CScript; struct Sections; @@ -133,9 +131,6 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); -/** Returns, given services flags, a list of humanly readable (known) network services */ -UniValue GetServicesNames(ServiceFlags services); - /** * Serializing JSON objects depends on the outer type. Only arrays and * dictionaries can be nested in json. The top-level outer type is "NONE". diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 71005cfb6e..2ca8dac48c 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -16,8 +16,7 @@ namespace { class TxInputStream { public: - TxInputStream(int nVersionIn, const unsigned char *txTo, size_t txToLen) : - m_version(nVersionIn), + TxInputStream(const unsigned char *txTo, size_t txToLen) : m_data(txTo), m_remaining(txToLen) {} @@ -48,9 +47,7 @@ public: return *this; } - int GetVersion() const { return m_version; } private: - const int m_version; const unsigned char* m_data; size_t m_remaining; }; @@ -84,8 +81,8 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP } try { - TxInputStream stream(PROTOCOL_VERSION, txTo, txToLen); - CTransaction tx(deserialize, stream); + TxInputStream stream(txTo, txToLen); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); std::vector<CTxOut> spent_outputs; if (spentOutputs != nullptr) { @@ -102,7 +99,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(tx, PROTOCOL_VERSION) != txToLen) + if (GetSerializeSize(TX_WITH_WITNESS(tx)) != txToLen) return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); // Regardless of the verification result, the tx did not error. diff --git a/src/span.h b/src/span.h index 7209d21a58..2e8da27cde 100644 --- a/src/span.h +++ b/src/span.h @@ -222,15 +222,32 @@ public: template <typename O> friend class Span; }; +// Return result of calling .data() method on type T. This is used to be able to +// write template deduction guides for the single-parameter Span constructor +// below that will work if the value that is passed has a .data() method, and if +// the data method does not return a void pointer. +// +// It is important to check for the void type specifically below, so the +// deduction guides can be used in SFINAE contexts to check whether objects can +// be converted to spans. If the deduction guides did not explicitly check for +// void, and an object was passed that returned void* from data (like +// std::vector<bool>), the template deduction would succeed, but the Span<void> +// object instantiation would fail, resulting in a hard error, rather than a +// SFINAE error. +// https://stackoverflow.com/questions/68759148/sfinae-to-detect-the-explicitness-of-a-ctad-deduction-guide +// https://stackoverflow.com/questions/16568986/what-happens-when-you-call-data-on-a-stdvectorbool +template<typename T> +using DataResult = std::remove_pointer_t<decltype(std::declval<T&>().data())>; + // Deduction guides for Span // For the pointer/size based and iterator based constructor: template <typename T, typename EndOrSize> Span(T*, EndOrSize) -> Span<T>; // For the array constructor: template <typename T, std::size_t N> Span(T (&)[N]) -> Span<T>; // For the temporaries/rvalue references constructor, only supporting const output. -template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T>, const std::remove_pointer_t<decltype(std::declval<T&&>().data())>>>; +template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T> && !std::is_void_v<DataResult<T&&>>, const DataResult<T&&>>>; // For (lvalue) references, supporting mutable output. -template <typename T> Span(T&) -> Span<std::remove_pointer_t<decltype(std::declval<T&>().data())>>; +template <typename T> Span(T&) -> Span<std::enable_if_t<!std::is_void_v<DataResult<T&>>, DataResult<T&>>>; /** Pop the last element off a span, and return a reference to that element. */ template <typename T> diff --git a/src/streams.h b/src/streams.h index d58de5233b..a3f8028da7 100644 --- a/src/streams.h +++ b/src/streams.h @@ -45,45 +45,6 @@ inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_off } } // namespace util -template<typename Stream> -class OverrideStream -{ - Stream* stream; - - const int nVersion; - -public: - OverrideStream(Stream* stream_, int nVersion_) : stream{stream_}, nVersion{nVersion_} {} - - template<typename T> - OverrideStream<Stream>& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return (*this); - } - - template<typename T> - OverrideStream<Stream>& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return (*this); - } - - void write(Span<const std::byte> src) - { - stream->write(src); - } - - void read(Span<std::byte> dst) - { - stream->read(dst); - } - - int GetVersion() const { return nVersion; } - size_t size() const { return stream->size(); } - void ignore(size_t size) { return stream->ignore(size); } -}; - /* Minimal stream for overwriting and/or appending to an existing byte vector * * The referenced vector will grow as necessary @@ -182,6 +143,11 @@ public: memcpy(dst.data(), m_data.data(), dst.size()); m_data = m_data.subspan(dst.size()); } + + void ignore(size_t n) + { + m_data = m_data.subspan(n); + } }; /** Double ended buffer combining vector and stream-like interfaces. @@ -471,7 +437,7 @@ class AutoFile { protected: std::FILE* m_file; - const std::vector<std::byte> m_xor; + std::vector<std::byte> m_xor; public: explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} @@ -511,6 +477,9 @@ public: */ bool IsNull() const { return m_file == nullptr; } + /** Continue with a different XOR key */ + void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; } + /** Implementation detail, only used internally. */ std::size_t detail_fread(Span<std::byte> dst); diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 4348a20886..e4ef019daf 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -51,8 +51,8 @@ static CBlock BuildBlockTestCase() { } // Number of shared use_counts we expect for a tx we haven't touched -// (block + mempool + our copy from the GetSharedTx call) -constexpr long SHARED_TX_OFFSET{3}; +// (block + mempool entry + mempool txns_randomized + our copy from the GetSharedTx call) +constexpr long SHARED_TX_OFFSET{4}; BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2], MemPoolRemovalReason::REPLACED); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock CBlock block2; { @@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that partialBlock = tmp; } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); @@ -196,15 +196,15 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 txhash = block.vtx[2]->GetHash(); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -240,7 +240,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; @@ -253,9 +253,9 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) txhash = block.vtx[1]->GetHash(); block.vtx.clear(); block2.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index c6800c498b..e1b31c20f4 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; - BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE); } BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain100Setup) diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 93c0412593..23cbe921ba 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -101,13 +101,13 @@ BOOST_AUTO_TEST_CASE(bloom_match) { // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) CDataStream stream(ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), SER_DISK, CLIENT_VERSION); - CTransaction tx(deserialize, stream); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx(deserialize, spendStream); + CTransaction spendingTx(deserialize, TX_WITH_WITNESS, spendStream); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); @@ -213,7 +213,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_2) // With 4 txes CBlock block; CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the first transaction @@ -268,7 +268,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_2_with_update_none) // With 4 txes CBlock block; CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the first transaction @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) // With one tx CBlock block; CDataStream stream(ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020a02ffffffff0100f2052a01000000434104ecd3229b0571c3be876feaac0442a9f13c5a572742927af1dc623353ecf8c202225f64868137a18cdd85cbbb4c74fbccfd4f49639cf1bdc94a5672bb15ad5d4cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the only transaction @@ -356,7 +356,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the last transaction @@ -402,7 +402,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_p2pubkey_only) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_P2PUBKEY_ONLY); // Match the generation pubkey @@ -425,7 +425,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the generation pubkey diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6e740a4f53..0fef8c5906 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -147,9 +147,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = MAX_FEELER_CONNECTIONS; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; const auto time_init{GetTime<std::chrono::seconds>()}; SetMockTime(time_init); @@ -248,9 +246,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; - options.m_max_outbound_block_relay = max_outbound_block_relay; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; connman->Init(options); std::vector<CNode*> vNodes; diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 3882e0e547..4a040c56de 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -63,17 +63,28 @@ FUZZ_TARGET(banman, .init = initialize_banman) // The complexity is O(N^2), where N is the input size, because each call // might call DumpBanlist (or other methods that are at least linear // complexity of the input size). + bool contains_invalid{false}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) { CallOneOf( fuzzed_data_provider, [&] { - ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CNetAddr net_addr{ConsumeNetAddr(fuzzed_data_provider)}; + const std::optional<CNetAddr>& addr{LookupHost(net_addr.ToStringAddr(), /*fAllowLookup=*/false)}; + if (addr.has_value() && addr->IsValid()) { + net_addr = *addr; + } else { + contains_invalid = true; + } + ban_man.Ban(net_addr, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { - ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CSubNet subnet{ConsumeSubNet(fuzzed_data_provider)}; + subnet = LookupSubNet(subnet.ToString()); + if (!subnet.IsValid()) { + contains_invalid = true; + } + ban_man.Ban(subnet, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { ban_man.ClearBanned(); @@ -109,7 +120,9 @@ FUZZ_TARGET(banman, .init = initialize_banman) BanMan ban_man_read{banlist_file, /*client_interface=*/nullptr, /*default_ban_time=*/0}; banmap_t banmap_read; ban_man_read.GetBanned(banmap_read); - assert(banmap == banmap_read); + if (!contains_invalid) { + assert(banmap == banmap_read); + } } } fs::remove(fs::PathToString(banlist_file + ".json")); diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 8c97fba323..09c0ed23d2 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -31,7 +31,7 @@ FUZZ_TARGET(block, .init = initialize_block) int nVersion; ds >> nVersion; ds.SetVersion(nVersion); - ds >> block; + ds >> TX_WITH_WITNESS(block); } catch (const std::ios_base::failure&) { return; } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 3e303ecc0f..5dbb463a16 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -10,21 +10,22 @@ #include <uint256.h> #include <cassert> -#include <cstdint> +#include <limits> #include <optional> -#include <string> #include <vector> FUZZ_TARGET(bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; CBloomFilter bloom_filter{ fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 10000000), 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.remaining_bytes() > 0, 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -37,6 +38,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { + good_data = false; return; } (void)bloom_filter.contains(*out_point); @@ -47,6 +49,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { + good_data = false; return; } (void)bloom_filter.contains(*u256); @@ -55,8 +58,9 @@ FUZZ_TARGET(bloom_filter) assert(present); }, [&] { - const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mut_tx) { + good_data = false; return; } const CTransaction tx{*mut_tx}; diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b088aa0bd7..8f3e357a84 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,26 +2,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> -#include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> -#include <pubkey.h> +#include <script/interpreter.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <util/hasher.h> +#include <cassert> #include <cstdint> #include <limits> +#include <memory> #include <optional> +#include <stdexcept> #include <string> +#include <utility> #include <vector> namespace { @@ -44,12 +46,15 @@ void initialize_coins_view() FUZZ_TARGET(coins_view, .init = initialize_coins_view) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; + CCoinsView backend_coins_view; CCoinsViewCache coins_view_cache{&backend_coins_view, /*deterministic=*/true}; COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -95,6 +100,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!opt_out_point) { + good_data = false; return; } random_out_point = *opt_out_point; @@ -102,13 +108,15 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } random_coin = *opt_coin; }, [&] { - const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_mutable_transaction) { + good_data = false; return; } random_mutable_transaction = *opt_mutable_transaction; @@ -116,7 +124,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { CCoinsMapMemoryResource resource; CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { @@ -124,6 +133,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } else { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } coins_cache_entry.coin = *opt_coin; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0dab2a2e97..551e1c526d 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -121,7 +121,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); }); } - (void)connman.GetAddedNodeInfo(); + (void)connman.GetAddedNodeInfo(fuzzed_data_provider.ConsumeBool()); (void)connman.GetExtraFullOutboundCount(); (void)connman.GetLocalServices(); (void)connman.GetMaxOutboundTarget(); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index abc230c07d..33c9a93c65 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -217,7 +217,7 @@ FUZZ_TARGET_DESERIALIZE(psbt_output_deserialize, { }) FUZZ_TARGET_DESERIALIZE(block_deserialize, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); }) FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { CBlockLocator bl; @@ -225,7 +225,7 @@ FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { }) FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); bool mutated; BlockMerkleRoot(block, &mutated); }) diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 0534f9bcf1..1f0fa5527a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -14,6 +14,11 @@ /** * Can be used to limit a theoretically unbounded loop. This caps the runtime * to avoid timeouts or OOMs. + * + * This can be used in combination with a check in the condition to confirm + * whether the fuzz engine provided "good" data. If the fuzz input contains + * invalid data, the loop aborts early. This will teach the fuzz engine to look + * for useful data and avoids bloating the fuzz input folder with useless data. */ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 6271367a9c..15e8db7536 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -27,7 +27,7 @@ FUZZ_TARGET(merkleblock) }, [&] { CMerkleBlock merkle_block; - const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); CBloomFilter bloom_filter; std::set<uint256> txids; if (opt_block && !opt_block->vtx.empty()) { diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 8d316134cc..8658c0b45a 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -40,7 +40,7 @@ void initialize_tx_pool() g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_EMPTY)}; if (i < COINBASE_MATURITY) { // Remember the txids to avoid expensive disk access later on g_outpoints_coinbase_init_mature.push_back(prevout); @@ -195,7 +195,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // Create input const auto sequence = ConsumeSequence(fuzzed_data_provider); const auto script_sig = CScript{}; - const auto script_wit_stack = std::vector<std::vector<uint8_t>>{WITNESS_STACK_ELEM_OP_TRUE}; + const auto script_wit_stack = fuzzed_data_provider.ConsumeBool() ? P2WSH_EMPTY_TRUE_STACK : P2WSH_EMPTY_TWO_STACK; + CTxIn in; in.prevout = outpoint; in.nSequence = sequence; @@ -204,17 +205,30 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) tx_mut.vin.push_back(in); } + + // Duplicate an input + bool dup_input = fuzzed_data_provider.ConsumeBool(); + if (dup_input) { + tx_mut.vin.push_back(tx_mut.vin.back()); + } + + // Refer to a non-existant input + if (fuzzed_data_provider.ConsumeBool()) { + tx_mut.vin.emplace_back(); + } + const auto amount_fee = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, amount_in); const auto amount_out = (amount_in - amount_fee) / num_out; for (int i = 0; i < num_out; ++i) { - tx_mut.vout.emplace_back(amount_out, P2WSH_OP_TRUE); + tx_mut.vout.emplace_back(amount_out, P2WSH_EMPTY); } // TODO vary transaction sizes to catch size-related issues auto tx = MakeTransactionRef(tx_mut); // Restore previously removed outpoints, except in-package outpoints if (!last_tx) { for (const auto& in : tx->vin) { - Assert(outpoints.insert(in.prevout).second); + // It's a fake input, or a new input, or a duplicate + Assert(in == CTxIn() || outpoints.insert(in.prevout).second || dup_input); } // Cache the in-package outpoints being made for (size_t i = 0; i < tx->vout.size(); ++i) { diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index ae7a68762e..4a4b46da60 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -44,7 +44,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider)}; + auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS)}; if (!block || block->vtx.size() == 0 || block->vtx.size() >= std::numeric_limits<uint16_t>::max()) { return; diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 227ee9d2c4..a6275d2fd2 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -6,16 +6,14 @@ #include <policy/fees.h> #include <policy/fees_args.h> #include <primitives/transaction.h> +#include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> -#include <txmempool.h> -#include <cstdint> #include <optional> -#include <string> #include <vector> namespace { @@ -31,13 +29,17 @@ void initialize_policy_estimator() FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; return; } const CTransaction tx{*mtx}; @@ -48,9 +50,11 @@ FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) }, [&] { std::vector<CTxMemPoolEntry> mempool_entries; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; break; } const CTransaction tx{*mtx}; diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 48815c8910..64f1c8cf8d 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -24,8 +24,8 @@ FUZZ_TARGET(primitives_transaction) const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); - const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mutable_tx_1 && mutable_tx_2) { const CTransaction tx_1{*mutable_tx_1}; const CTransaction tx_2{*mutable_tx_2}; diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index dbe99029c3..aa6385d12d 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -33,7 +33,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { return; } @@ -42,7 +42,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!another_mtx) { break; } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e2..97b99ea2a3 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -3,18 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <base58.h> -#include <core_io.h> #include <key.h> #include <key_io.h> -#include <node/context.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> -#include <rpc/blockchain.h> #include <rpc/client.h> #include <rpc/request.h> #include <rpc/server.h> -#include <rpc/util.h> #include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -22,19 +18,23 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <tinyformat.h> +#include <uint256.h> #include <univalue.h> -#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> +#include <algorithm> +#include <cassert> #include <cstdint> +#include <cstdlib> +#include <exception> #include <iostream> #include <memory> #include <optional> #include <stdexcept> -#include <string> #include <vector> +enum class ChainType; namespace { struct RPCFuzzTestingSetup : public TestingSetup { @@ -184,7 +184,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "waitfornewblock", }; -std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { const size_t max_string_length = 4096; const size_t max_base58_bytes_length{64}; @@ -249,18 +249,20 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded block - std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_block) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; - data_stream << *opt_block; + data_stream << TX_WITH_WITNESS(*opt_block); r = HexStr(data_stream); }, [&] { // hex encoded block header std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!opt_block_header) { + good_data = false; return; } DataStream data_stream{}; @@ -269,18 +271,21 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded tx - std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_tx) { + good_data = false; return; } - CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; - data_stream << *opt_tx; + DataStream data_stream; + auto allow_witness = (fuzzed_data_provider.ConsumeBool() ? TX_WITH_WITNESS : TX_NO_WITNESS); + data_stream << allow_witness(*opt_tx); r = HexStr(data_stream); }, [&] { // base64 encoded psbt std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); if (!opt_psbt) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; @@ -291,6 +296,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // base58 encoded key CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = EncodeSecret(key); @@ -299,6 +305,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded pubkey CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = HexStr(key.GetPubKey()); @@ -306,18 +313,19 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) return r; } -std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { std::vector<std::string> scalar_arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider, good_data)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; } -std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { - return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider, good_data) : ConsumeArrayRPCArgument(fuzzed_data_provider, good_data); } RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() @@ -353,6 +361,7 @@ void initialize_rpc() FUZZ_TARGET(rpc, .init = initialize_rpc) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { @@ -363,8 +372,9 @@ FUZZ_TARGET(rpc, .init = initialize_rpc) return; } std::vector<std::string> arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider, good_data)); } try { rpc_testing_setup->CallRPC(rpc_command, arguments); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 66c862a6f9..55a4fbc75c 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -54,7 +54,7 @@ CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; try { - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; + SpanReader{0, CheckedParseHex(str)} >> TX_NO_WITNESS(tx); } catch (const std::ios_base::failure&) { throw std::runtime_error("Tx deserialization failure"); } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index f8594fc233..3b8f5c068d 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -23,7 +23,7 @@ FUZZ_TARGET(script_flags) } try { - const CTransaction tx(deserialize, ds); + const CTransaction tx(deserialize, TX_WITH_WITNESS, ds); unsigned int verify_flags; ds >> verify_flags; diff --git a/src/test/fuzz/script_interpreter.cpp b/src/test/fuzz/script_interpreter.cpp index 5d59771682..5e76443abe 100644 --- a/src/test/fuzz/script_interpreter.cpp +++ b/src/test/fuzz/script_interpreter.cpp @@ -20,13 +20,13 @@ FUZZ_TARGET(script_interpreter) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); { const CScript script_code = ConsumeScript(fuzzed_data_provider); - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx) { const CTransaction tx_to{*mtx}; const unsigned int in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (in < tx_to.vin.size()) { (void)SignatureHash(script_code, tx_to, in, fuzzed_data_provider.ConsumeIntegral<int>(), ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}), nullptr); - const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx_precomputed) { const CTransaction tx_precomputed{*mtx_precomputed}; const PrecomputedTransactionData precomputed_transaction_data{tx_precomputed}; diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 486b1e5197..5fdbc9e106 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -30,7 +30,7 @@ FUZZ_TARGET(script_sigcache, .init = initialize_script_sigcache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const CTransaction tx{mutable_transaction ? *mutable_transaction : CMutableTransaction{}}; const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const CAmount amount = ConsumeMoney(fuzzed_data_provider); diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 179715b6ea..0944c91c4a 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -86,7 +86,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) } { - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const std::optional<CTxOut> tx_out = ConsumeDeserializable<CTxOut>(fuzzed_data_provider); const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (mutable_transaction && tx_out && mutable_transaction->vin.size() > n_in) { @@ -100,7 +100,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) if (mutable_transaction) { CTransaction tx_from{*mutable_transaction}; CMutableTransaction tx_to; - const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (opt_tx_to) { tx_to = *opt_tx_to; } diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 3ccf5eb6f0..6f1996572e 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -25,7 +25,7 @@ void initialize_signet() FUZZ_TARGET(signet, .init = initialize_signet) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!block) { return; } diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 05b8312ab2..af81fcb593 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -32,7 +32,9 @@ FUZZ_TARGET(socks5, .init = initialize_socks5) ProxyCredentials proxy_credentials; proxy_credentials.username = fuzzed_data_provider.ConsumeRandomLengthString(512); proxy_credentials.password = fuzzed_data_provider.ConsumeRandomLengthString(512); - InterruptSocks5(fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + g_socks5_interrupt(); + } // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 88c2a334c7..ccaee424fb 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -40,7 +40,7 @@ FUZZ_TARGET(transaction, .init = initialize_transaction) bool valid_tx = true; const CTransaction tx = [&] { try { - return CTransaction(deserialize, ds); + return CTransaction(deserialize, TX_WITH_WITNESS, ds); } catch (const std::ios_base::failure&) { valid_tx = false; return CTransaction{CMutableTransaction{}}; @@ -53,7 +53,7 @@ FUZZ_TARGET(transaction, .init = initialize_transaction) int nVersion; ds_mtx >> nVersion; ds_mtx.SetVersion(nVersion); - ds_mtx >> mutable_tx; + ds_mtx >> TX_WITH_WITNESS(mutable_tx); } catch (const std::ios_base::failure&) { valid_mutable_tx = false; } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 66e537a57b..96095539ec 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -154,12 +154,15 @@ void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, b // It may be already in the mempool since in ATMP cases we don't set MEMPOOL_ENTRY or DIFFERENT_WITNESS Assert(!res.m_state.IsValid()); Assert(res.m_state.IsInvalid()); + + const bool is_reconsiderable{res.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; Assert(!res.m_replaced_transactions); Assert(!res.m_vsize); Assert(!res.m_base_fees); - // Unable or unwilling to calculate fees - Assert(!res.m_effective_feerate); - Assert(!res.m_wtxids_fee_calculations); + // Fee information is provided if the failure is TX_RECONSIDERABLE. + // In other cases, validation may be unable or unwilling to calculate the fees. + Assert(res.m_effective_feerate.has_value() == is_reconsiderable); + Assert(res.m_wtxids_fee_calculations.has_value() == is_reconsiderable); Assert(!res.m_other_wtxid); break; } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index 318797faf2..a8f0f3b7ff 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -94,8 +94,8 @@ FUZZ_TARGET(utxo_total_supply) assert(ActiveHeight() == 0); // Get at which height we duplicate the coinbase // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. - // Up to 2000 seems reasonable. - int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Up to 300 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 300); // Always pad with OP_0 at the end to avoid bad-cb-length error const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; // Mine the first block with this duplicate @@ -121,7 +121,7 @@ FUZZ_TARGET(utxo_total_supply) // Limit to avoid timeout, but enough to cover duplicate_coinbase_height // and CVE-2018-17144. - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'00) { CallOneOf( fuzzed_data_provider, diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 54afcef989..f0d2b9ed72 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,12 +122,12 @@ BOOST_AUTO_TEST_CASE(siphash) (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } - CHashWriter ss{CLIENT_VERSION}; + HashWriter ss{}; CMutableTransaction tx; // Note these tests were originally written with tx.nVersion=1 // and the test would be affected by default tx version bumps if not fixed. tx.nVersion = 1; - ss << tx; + ss << TX_WITH_WITNESS(tx); BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); // Check consistency between CSipHasher and SipHashUint256[Extra]. diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index db58a0baec..217e4a6d22 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort<descendant_score>(pool, sortedOrder); CTxMemPool::setEntries setAncestors; - setAncestors.insert(pool.mapTx.find(tx6.GetHash())); + setAncestors.insert(pool.GetIter(tx6.GetHash()).value()); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout.resize(1); tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; - setAncestors.insert(pool.mapTx.find(tx7.GetHash())); + setAncestors.insert(pool.GetIter(tx7.GetHash()).value()); pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high @@ -247,8 +247,8 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> snapshotOrder = sortedOrder; - setAncestors.insert(pool.mapTx.find(tx8.GetHash())); - setAncestors.insert(pool.mapTx.find(tx9.GetHash())); + setAncestors.insert(pool.GetIter(tx8.GetHash()).value()); + setAncestors.insert(pool.GetIter(tx9.GetHash()).value()); /* tx10 depends on tx8 and tx9 and has a high fee*/ CMutableTransaction tx10 = CMutableTransaction(); tx10.vin.resize(2); @@ -291,11 +291,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10U); // Now try removing tx10 and verify the sort order returns to normal - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx10.GetHash())), REMOVAL_REASON_DUMMY); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx9.GetHash())), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx8.GetHash())), REMOVAL_REASON_DUMMY); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index b4c7cac223..96acbea317 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -144,7 +144,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee uint256 hashFreeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).FromTx(tx)); - size_t freeTxSize = ::GetSerializeSize(tx, PROTOCOL_VERSION); + size_t freeTxSize = ::GetSerializeSize(TX_WITH_WITNESS(tx)); // Calculate a fee on child transaction that will put the package just // below the block min tx fee (assuming 1 child tx of the same size). diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index 76c079a6e7..311e402e3e 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -94,7 +94,7 @@ BOOST_FIXTURE_TEST_CASE(miniminer_negative, TestChain100Setup) const CFeeRate feerate_zero(0); mini_miner_target0.BuildMockTemplate(feerate_zero); // Check the quit condition: - BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(pool.GetIter(tx_mod_negative->GetHash()).value()->GetTxSize())); + BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(Assert(pool.GetEntry(tx_mod_negative->GetHash()))->GetTxSize())); BOOST_CHECK(mini_miner_target0.GetMockTemplateTxids().empty()); // With no target feerate, the template includes all transactions, even negative feerate ones. @@ -179,9 +179,9 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) }; std::map<uint256, TxDimensions> tx_dims; for (const auto& tx : all_transactions) { - const auto it = pool.GetIter(tx->GetHash()).value(); - tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), - CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + const auto& entry{*Assert(pool.GetEntry(tx->GetHash()))}; + tx_dims.emplace(tx->GetHash(), TxDimensions{entry.GetTxSize(), entry.GetModifiedFee(), + CFeeRate(entry.GetModifiedFee(), entry.GetTxSize())}); } const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), @@ -325,14 +325,14 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) const int32_t tx6_vsize{tx_dims.at(tx6->GetHash()).vsize}; const int32_t tx7_vsize{tx_dims.at(tx7->GetHash()).vsize}; - miniminer_info.emplace_back(/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize, tx0); - miniminer_info.emplace_back( med_fee, 2*med_fee, tx1_vsize, tx0_vsize + tx1_vsize, tx1); - miniminer_info.emplace_back( low_fee, low_fee, tx2_vsize, tx2_vsize, tx2); - miniminer_info.emplace_back( high_fee, low_fee + high_fee, tx3_vsize, tx2_vsize + tx3_vsize, tx3); - miniminer_info.emplace_back( low_fee, low_fee, tx4_vsize, tx4_vsize, tx4); - miniminer_info.emplace_back( tx5_mod_fee, low_fee + tx5_mod_fee, tx5_vsize, tx4_vsize + tx5_vsize, tx5); - miniminer_info.emplace_back( high_fee, high_fee, tx6_vsize, tx6_vsize, tx6); - miniminer_info.emplace_back( low_fee, high_fee + low_fee, tx7_vsize, tx6_vsize + tx7_vsize, tx7); + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize,/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee); + miniminer_info.emplace_back(tx1, tx1_vsize, tx0_vsize + tx1_vsize, med_fee, 2*med_fee); + miniminer_info.emplace_back(tx2, tx2_vsize, tx2_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx3, tx3_vsize, tx2_vsize + tx3_vsize, high_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx4, tx4_vsize, tx4_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx5, tx5_vsize, tx4_vsize + tx5_vsize, tx5_mod_fee, low_fee + tx5_mod_fee); + miniminer_info.emplace_back(tx6, tx6_vsize, tx6_vsize, high_fee, high_fee); + miniminer_info.emplace_back(tx7, tx7_vsize, tx6_vsize + tx7_vsize, low_fee, high_fee + low_fee); } std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx1->GetHash()}); @@ -447,15 +447,15 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) // tx3's feerate is lower than tx2's. same fee, different weight. BOOST_CHECK(tx2_feerate > tx3_feerate); const auto tx3_anc_feerate = CFeeRate(low_fee + med_fee + high_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]); - const auto tx3_iter = pool.GetIter(tx3->GetHash()); - BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_iter.value()->GetModFeesWithAncestors(), tx3_iter.value()->GetSizeWithAncestors())); + const auto& tx3_entry{*Assert(pool.GetEntry(tx3->GetHash()))}; + BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_entry.GetModFeesWithAncestors(), tx3_entry.GetSizeWithAncestors())); const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[4]); const auto tx6_anc_feerate = CFeeRate(high_fee + low_fee + med_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]); - const auto tx6_iter = pool.GetIter(tx6->GetHash()); - BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_iter.value()->GetModFeesWithAncestors(), tx6_iter.value()->GetSizeWithAncestors())); + const auto& tx6_entry{*Assert(pool.GetEntry(tx6->GetHash()))}; + BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_entry.GetModFeesWithAncestors(), tx6_entry.GetSizeWithAncestors())); const auto tx7_anc_feerate = CFeeRate(high_fee + low_fee + high_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]); - const auto tx7_iter = pool.GetIter(tx7->GetHash()); - BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_iter.value()->GetModFeesWithAncestors(), tx7_iter.value()->GetSizeWithAncestors())); + const auto& tx7_entry{*Assert(pool.GetEntry(tx7->GetHash()))}; + BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_entry.GetModFeesWithAncestors(), tx7_entry.GetSizeWithAncestors())); BOOST_CHECK(tx4_feerate > tx6_anc_feerate); BOOST_CHECK(tx4_feerate > tx7_anc_feerate); @@ -543,15 +543,14 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) } // Check linearization order std::vector<node::MiniMinerMempoolEntry> miniminer_info; - miniminer_info.emplace_back(/*fee_self=*/low_fee, /*fee_ancestor=*/low_fee,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], tx0); - miniminer_info.emplace_back( med_fee, med_fee, tx_vsizes[1], tx_vsizes[1], tx1); - miniminer_info.emplace_back( high_fee, high_fee, tx_vsizes[2], tx_vsizes[2], tx2); - miniminer_info.emplace_back( high_fee, low_fee+med_fee+2*high_fee, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], tx3); - - miniminer_info.emplace_back( high_fee, high_fee, tx_vsizes[4], tx_vsizes[4], tx4); - miniminer_info.emplace_back( low_fee, low_fee + high_fee, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], tx5); - miniminer_info.emplace_back( med_fee, high_fee+low_fee+med_fee, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], tx6); - miniminer_info.emplace_back( high_fee, high_fee+low_fee+high_fee, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], tx7); + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], /*fee_self=*/low_fee, /*fee_ancestor=*/low_fee); + miniminer_info.emplace_back(tx1, tx_vsizes[1], tx_vsizes[1], med_fee, med_fee); + miniminer_info.emplace_back(tx2, tx_vsizes[2], tx_vsizes[2], high_fee, high_fee); + miniminer_info.emplace_back(tx3, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], high_fee, low_fee+med_fee+2*high_fee); + miniminer_info.emplace_back(tx4, tx_vsizes[4], tx_vsizes[4], high_fee, high_fee); + miniminer_info.emplace_back(tx5, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], low_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx6, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], med_fee, high_fee+low_fee+med_fee); + miniminer_info.emplace_back(tx7, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], high_fee, high_fee+low_fee+high_fee); std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx3->GetHash()}); @@ -647,7 +646,7 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) CTxMemPool& pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); { - // 3 pairs of fee-bumping grandparent + parent, plus 1 low-feerate child. + // 3 pairs of grandparent + fee-bumping parent, plus 1 low-feerate child. // 0 fee + high fee auto grandparent_zero_fee = make_tx({{m_coinbase_txns.at(0)->GetHash(), 0}}, 1); auto parent_high_feerate = make_tx({{grandparent_zero_fee->GetHash(), 0}}, 1); @@ -665,13 +664,13 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) const int64_t child_vsize{1000}; std::vector<node::MiniMinerMempoolEntry> miniminer_info; - miniminer_info.emplace_back(/*fee_self=*/0, /*fee_ancestor=*/0,/*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, grandparent_zero_fee); - miniminer_info.emplace_back( high_fee, high_fee, tx_vsize, 2*tx_vsize, parent_high_feerate); - miniminer_info.emplace_back( 2*low_fee, 2*low_fee, tx_vsize, tx_vsize, grandparent_double_low_feerate); - miniminer_info.emplace_back( med_fee, 2*low_fee+med_fee, tx_vsize, 2*tx_vsize, parent_med_feerate); - miniminer_info.emplace_back( low_fee, low_fee, tx_vsize, tx_vsize, grandparent_low_feerate); - miniminer_info.emplace_back( 2*low_fee, 3*low_fee, tx_vsize, 2*tx_vsize, parent_double_low_feerate); - miniminer_info.emplace_back( low_fee, high_fee+med_fee+6*low_fee, child_vsize, 6*tx_vsize+child_vsize, child); + miniminer_info.emplace_back(grandparent_zero_fee, /*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, /*fee_self=*/0,/*fee_ancestor=*/0); + miniminer_info.emplace_back(parent_high_feerate, tx_vsize, 2*tx_vsize, high_fee, high_fee); + miniminer_info.emplace_back(grandparent_double_low_feerate, tx_vsize, tx_vsize, 2*low_fee, 2*low_fee); + miniminer_info.emplace_back(parent_med_feerate, tx_vsize, 2*tx_vsize, med_fee, 2*low_fee+med_fee); + miniminer_info.emplace_back(grandparent_low_feerate, tx_vsize, tx_vsize, low_fee, low_fee); + miniminer_info.emplace_back(parent_double_low_feerate, tx_vsize, 2*tx_vsize, 2*low_fee, 3*low_fee); + miniminer_info.emplace_back(child, child_vsize, 6*tx_vsize+child_vsize, low_fee, high_fee+med_fee+6*low_fee); std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(grandparent_zero_fee->GetHash(), std::set<Txid>{grandparent_zero_fee->GetHash(), parent_high_feerate->GetHash(), child->GetHash()}); descendant_caches.emplace(grandparent_low_feerate->GetHash(), std::set<Txid>{grandparent_low_feerate->GetHash(), parent_double_low_feerate->GetHash(), child->GetHash()}); @@ -693,7 +692,7 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) BOOST_CHECK_EQUAL(sequences.at(grandparent_double_low_feerate->GetHash()), 1); BOOST_CHECK_EQUAL(sequences.at(parent_med_feerate->GetHash()), 1); - // CPFP low + med + // CPFP low + double low BOOST_CHECK_EQUAL(sequences.at(grandparent_low_feerate->GetHash()), 2); BOOST_CHECK_EQUAL(sequences.at(parent_double_low_feerate->GetHash()), 2); diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp new file mode 100644 index 0000000000..3d3f296d82 --- /dev/null +++ b/src/test/net_peer_connection_tests.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compat/compat.h> +#include <net.h> +#include <net_processing.h> +#include <netaddress.h> +#include <netbase.h> +#include <netgroup.h> +#include <node/connection_types.h> +#include <protocol.h> +#include <random.h> +#include <test/util/logging.h> +#include <test/util/net.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <util/chaintype.h> +#include <version.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <boost/test/unit_test.hpp> + +struct LogIPsTestingSetup : public TestingSetup { + LogIPsTestingSetup() + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-logips"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(net_peer_connection_tests, LogIPsTestingSetup) + +static CService ip(uint32_t i) +{ + struct in_addr s; + s.s_addr = i; + return CService{CNetAddr{s}, Params().GetDefaultPort()}; +} + +/** Create a peer and connect to it. If the optional `address` (IP/CJDNS only) isn't passed, a random address is created. */ +static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman, ConnmanTestMsg& connman, ConnectionType conn_type, bool onion_peer = false, std::optional<std::string> address = std::nullopt) +{ + CAddress addr{}; + + if (address.has_value()) { + addr = CAddress{MaybeFlipIPv6toCJDNS(LookupNumeric(address.value(), Params().GetDefaultPort())), NODE_NONE}; + } else if (onion_peer) { + auto tor_addr{g_insecure_rand_ctx.randbytes(ADDR_TORV3_SIZE)}; + BOOST_REQUIRE(addr.SetSpecial(OnionToString(tor_addr))); + } + + while (!addr.IsLocal() && !addr.IsRoutable()) { + addr = CAddress{ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE}; + } + + BOOST_REQUIRE(addr.IsValid()); + + const bool inbound_onion{onion_peer && conn_type == ConnectionType::INBOUND}; + + nodes.emplace_back(new CNode{++id, + /*sock=*/nullptr, + addr, + /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, + CAddress{}, + /*addrNameIn=*/"", + conn_type, + /*inbound_onion=*/inbound_onion}); + CNode& node = *nodes.back(); + node.SetCommonVersion(PROTOCOL_VERSION); + + peerman.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node.fSuccessfullyConnected = true; + + connman.AddTestNode(node); +} + +BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) +{ + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + NodeId id{0}; + std::vector<CNode*> nodes; + + // Connect a localhost peer. + { + ASSERT_DEBUG_LOG("Added connection to 127.0.0.1:8333 peer=1"); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::MANUAL, /*onion_peer=*/false, /*address=*/"127.0.0.1"); + BOOST_REQUIRE(nodes.back() != nullptr); + } + + // Call ConnectNode(), which is also called by RPC addnode onetry, for a localhost + // address that resolves to multiple IPs, including that of the connected peer. + // The connection attempt should consistently fail due to the check in ConnectNode(). + for (int i = 0; i < 10; ++i) { + ASSERT_DEBUG_LOG("Not opening a connection to localhost, already connected to 127.0.0.1:8333"); + BOOST_CHECK(!connman->ConnectNodePublic(*peerman, "localhost", ConnectionType::MANUAL)); + } + + // Add 3 more peer connections. + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::BLOCK_RELAY, /*onion_peer=*/true); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND); + + BOOST_TEST_MESSAGE("Call AddNode() for all the peers"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddNode({/*m_added_node=*/node->addr.ToStringAddrPort(), /*m_use_v2transport=*/true})); + BOOST_TEST_MESSAGE(strprintf("peer id=%s addr=%s", node->GetId(), node->addr.ToStringAddrPort())); + } + + BOOST_TEST_MESSAGE("\nCall AddNode() with 2 addrs resolving to existing localhost addnode entry; neither should be added"); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.0.0.1", /*m_use_v2transport=*/true})); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.1", /*m_use_v2transport=*/true})); + + BOOST_TEST_MESSAGE("\nExpect GetAddedNodeInfo to return expected number of peers with `include_connected` true/false"); + BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); + BOOST_CHECK(connman->GetAddedNodeInfo(/*include_connected=*/false).empty()); + + BOOST_TEST_MESSAGE("\nPrint GetAddedNodeInfo contents:"); + for (const auto& info : connman->GetAddedNodeInfo(/*include_connected=*/true)) { + BOOST_TEST_MESSAGE(strprintf("\nadded node: %s", info.m_params.m_added_node)); + BOOST_TEST_MESSAGE(strprintf("connected: %s", info.fConnected)); + if (info.fConnected) { + BOOST_TEST_MESSAGE(strprintf("IP address: %s", info.resolvedAddress.ToStringAddrPort())); + BOOST_TEST_MESSAGE(strprintf("direction: %s", info.fInbound ? "inbound" : "outbound")); + } + } + + BOOST_TEST_MESSAGE("\nCheck that all connected peers are correctly detected as connected"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AlreadyConnectedPublic(node->addr)); + } + + // Clean up + for (auto node : connman->TestNodes()) { + peerman->FinalizeNode(*node); + } + connman->ClearTestNodes(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 6e0c962601..874f3e51f7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -141,8 +141,8 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript } #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx2; + DataStream stream; + stream << TX_WITH_WITNESS(tx2); uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; if (libconsensus_flags == flags) { int expectedSuccessCode = expect ? 1 : 0; @@ -1470,7 +1470,7 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) static CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; + SpanReader{0, ParseHex(str)} >> TX_NO_WITNESS(tx); return tx; } @@ -1513,8 +1513,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1536,8 +1536,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1559,8 +1559,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size() * 2, nIn, libconsensus_flags, &err); @@ -1582,7 +1582,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream; stream << 0xffffffff; bitcoinconsensus_error err; @@ -1605,8 +1605,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1628,8 +1628,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1651,8 +1651,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_spent_outputs_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result{bitcoinconsensus_verify_script_with_spent_outputs(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nullptr, 0, nIn, libconsensus_flags, &err)}; @@ -1718,8 +1718,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1752,8 +1752,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1816,7 +1816,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) for (const auto& vec : vectors.getValues()) { auto txhex = ParseHex(vec["given"]["rawUnsignedTx"].get_str()); CMutableTransaction tx; - SpanReader{PROTOCOL_VERSION, txhex} >> tx; + SpanReader{PROTOCOL_VERSION, txhex} >> TX_WITH_WITNESS(tx); std::vector<CTxOut> utxos; for (const auto& utxo_spent : vec["given"]["utxosSpent"].getValues()) { auto script_bytes = ParseHex(utxo_spent["scriptPubKey"].get_str()); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 49827be559..6d5a14ad9e 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -36,7 +36,7 @@ public: READWRITE(obj.boolval); READWRITE(obj.stringval); READWRITE(obj.charstrval); - READWRITE(obj.txval); + READWRITE(TX_WITH_WITNESS(obj.txval)); } bool operator==(const CSerializeMethodsTestSingle& rhs) const @@ -56,7 +56,7 @@ public: SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) { - READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, TX_WITH_WITNESS(obj.txval)); } }; @@ -85,6 +85,8 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 1>{0}, 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 2>{0, 0}, 0), 2U); } BOOST_AUTO_TEST_CASE(varints) @@ -179,6 +181,16 @@ BOOST_AUTO_TEST_CASE(vector_bool) BOOST_CHECK((HashWriter{} << vec1).GetHash() == (HashWriter{} << vec2).GetHash()); } +BOOST_AUTO_TEST_CASE(array) +{ + std::array<uint8_t, 32> array1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + DataStream ds; + ds << array1; + std::array<uint8_t, 32> array2; + ds >> array2; + BOOST_CHECK(array1 == array2); +} + BOOST_AUTO_TEST_CASE(noncanonical) { // Write some non-canonical CompactSize encodings, and @@ -228,7 +240,7 @@ BOOST_AUTO_TEST_CASE(class_methods) CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, tx_ref); CSerializeMethodsTestSingle methodtest3; CSerializeMethodsTestMany methodtest4; - CDataStream ss(SER_DISK, PROTOCOL_VERSION); + DataStream ss; BOOST_CHECK(methodtest1 == methodtest2); ss << methodtest1; ss >> methodtest4; @@ -238,8 +250,8 @@ BOOST_AUTO_TEST_CASE(class_methods) BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); - CDataStream ss2{SER_DISK, PROTOCOL_VERSION}; - ss2 << intval << boolval << stringval << charstrval << txval; + DataStream ss2; + ss2 << intval << boolval << stringval << charstrval << TX_WITH_WITNESS(txval); ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); { diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 9653edd84b..bc2e3ca45e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -78,8 +78,8 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un } // Serialize and hash - CHashWriter ss{SERIALIZE_TRANSACTION_NO_WITNESS}; - ss << txTmp << nHashType; + HashWriter ss{}; + ss << TX_NO_WITNESS(txTmp) << nHashType; return ss.GetHash(); } @@ -138,8 +138,8 @@ BOOST_AUTO_TEST_CASE(sighash_test) sho = SignatureHashOld(scriptCode, CTransaction(txTo), nIn, nHashType); sh = SignatureHash(scriptCode, txTo, nIn, nHashType, 0, SigVersion::BASE); #if defined(PRINT_SIGHASH_JSON) - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << txTo; + DataStream ss; + ss << TX_WITH_WITNESS(txTo); std::cout << "\t[\"" ; std::cout << HexStr(ss) << "\", \""; @@ -188,8 +188,8 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) nHashType = test[3].getInt<int>(); sigHashHex = test[4].get_str(); - CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); - stream >> tx; + DataStream stream(ParseHex(raw_tx)); + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); diff --git a/src/test/span_tests.cpp b/src/test/span_tests.cpp new file mode 100644 index 0000000000..f6cac10b09 --- /dev/null +++ b/src/test/span_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> + +#include <boost/test/unit_test.hpp> +#include <array> +#include <set> +#include <vector> + +namespace { +struct Ignore +{ + template<typename T> Ignore(T&&) {} +}; +template<typename T> +bool Spannable(T&& value, decltype(Span{value})* enable = nullptr) +{ + return true; +} +bool Spannable(Ignore) +{ + return false; +} + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wunneeded-member-function" +# pragma clang diagnostic ignored "-Wunused-member-function" +#endif +struct SpannableYes +{ + int* data(); + size_t size(); +}; +struct SpannableNo +{ + void* data(); + size_t size(); +}; +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +} // namespace + +BOOST_AUTO_TEST_SUITE(span_tests) + +// Make sure template Span template deduction guides accurately enable calls to +// Span constructor overloads that work, and disable calls to constructor overloads that +// don't work. This makes it is possible to use the Span constructor in a SFINAE +// contexts like in the Spannable function above to detect whether types are or +// aren't compatible with Spans at compile time. +// +// Previously there was a bug where writing a SFINAE check for vector<bool> was +// not possible, because in libstdc++ vector<bool> has a data() memeber +// returning void*, and the Span template guide ignored the data() return value, +// so the template substitution would succeed, but the constructor would fail, +// resulting in a fatal compile error, rather than a SFINAE error that could be +// handled. +BOOST_AUTO_TEST_CASE(span_constructor_sfinae) +{ + BOOST_CHECK(Spannable(std::vector<int>{})); + BOOST_CHECK(!Spannable(std::set<int>{})); + BOOST_CHECK(!Spannable(std::vector<bool>{})); + BOOST_CHECK(Spannable(std::array<int, 3>{})); + BOOST_CHECK(Spannable(Span<int>{})); + BOOST_CHECK(Spannable("char array")); + BOOST_CHECK(Spannable(SpannableYes{})); + BOOST_CHECK(!Spannable(SpannableNo{})); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index a4c0db8aea..220b61f7e7 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -233,8 +233,8 @@ BOOST_AUTO_TEST_CASE(tx_valid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); @@ -321,8 +321,8 @@ BOOST_AUTO_TEST_CASE(tx_invalid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; if (!CheckTransaction(tx, state) || state.IsInvalid()) { @@ -371,9 +371,9 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) // Random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); - CDataStream stream(vch, SER_DISK, CLIENT_VERSION); + DataStream stream(vch); CMutableTransaction tx; - stream >> tx; + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid."); @@ -418,9 +418,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const outputm.vout.resize(1); outputm.vout[0].nValue = 1; outputm.vout[0].scriptPubKey = outscript; - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << outputm; - ssout >> output; + DataStream ssout; + ssout << TX_WITH_WITNESS(outputm); + ssout >> TX_WITH_WITNESS(output); assert(output->vin.size() == 1); assert(output->vin[0] == outputm.vin[0]); assert(output->vout.size() == 1); @@ -437,9 +437,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const SignatureData empty; bool ret = SignSignature(keystore, *output, inputm, 0, SIGHASH_ALL, empty); assert(ret == success); - CDataStream ssin(SER_NETWORK, PROTOCOL_VERSION); - ssin << inputm; - ssin >> input; + DataStream ssin; + ssin << TX_WITH_WITNESS(inputm); + ssin >> TX_WITH_WITNESS(input); assert(input.vin.size() == 1); assert(input.vin[0] == inputm.vin[0]); assert(input.vout.size() == 1); @@ -524,9 +524,9 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) assert(hashSigned); } - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << mtx; - CTransaction tx(deserialize, ssout); + DataStream ssout; + ssout << TX_WITH_WITNESS(mtx); + CTransaction tx(deserialize, TX_WITH_WITNESS, ssout); // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index debefa7f93..84c9ecc3d1 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -11,6 +11,7 @@ #include <test/util/random.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -132,36 +133,35 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_destination=*/child_locking_script, /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - BOOST_CHECK(result_parent_child.m_tx_results.size() == 2); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + Package package_parent_child{tx_parent, tx_child}; + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + + BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); + BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + } // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /*test_accept=*/true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - BOOST_CHECK(result_single_large.m_tx_results.size() == 1); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + Package package_single_giant{giant_ptx}; + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true); + if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { + BOOST_ERROR(err_single_large.value()); + } else { + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + } // Check that mempool size hasn't changed. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); @@ -281,6 +281,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_unrelated, /*test_accept=*/false); + // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -336,20 +337,20 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CMutableTransaction mtx_parent_invalid{mtx_parent}; mtx_parent_invalid.vin[0].scriptWitness = bad_witness; CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); + Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {tx_parent_invalid, tx_child}, /*test_accept=*/ false); - BOOST_CHECK(result_quit_early.m_state.IsInvalid()); + package_invalid_parent, /*test_accept=*/ false); + if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_parent_invalid.value()); + } else { + auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); + auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); + } BOOST_CHECK_EQUAL(result_quit_early.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(!result_quit_early.m_tx_results.empty()); - BOOST_CHECK_EQUAL(result_quit_early.m_tx_results.size(), 2); - auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); - auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_quit_early.m_tx_results.end()); - BOOST_CHECK(it_child != result_quit_early.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); } // Child with missing parent. @@ -381,36 +382,27 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_parent))); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != submit_parent_child.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_state.IsValid()); BOOST_CHECK(it_child->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_child))); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_deduped.m_tx_results.size(), package_parent_child.size()); - auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_parent_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_child_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_deduped.value()); + } else { + auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } } @@ -470,51 +462,39 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are // same-txid-different-witness. { + Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness1.m_tx_results.size(), 2); - auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child1 = submit_witness1.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_parent1->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_child1->second.m_state.GetRejectReason()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + package_parent_child1, /*test_accept=*/false); + if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness1.value()); + } // Child2 would have been validated individually. + Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness2.m_tx_results.size(), 2); - auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); - BOOST_CHECK(it_parent2_deduped != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child2 != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + package_parent_child2, /*test_accept=*/false); + if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness2.value()); + } else { + auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); + BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); + } // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_segwit_dedup.m_tx_results.size(), 2); - auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + package_parent_child1, /*test_accept=*/false); + if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_segwit_dedup.value()); + } else { + auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); + BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } } // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as @@ -535,21 +515,17 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // We already submitted child1 above. { + Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_spend_ignored.m_tx_results.size(), 2); - auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); - auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); - BOOST_CHECK(it_child2_ignored != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_grandchild != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + package_child2_grandchild, /*test_accept=*/false); + if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_spend_ignored.value()); + } else { + auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); + auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); + BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + } } // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of @@ -641,36 +617,28 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // child should be accepted { const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); - BOOST_CHECK_MESSAGE(mixed_result.m_state.IsValid(), mixed_result.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(mixed_result.m_tx_results.size(), package_mixed.size()); - auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); - auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); - auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); - auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); - BOOST_CHECK(it_parent1 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent2 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent3 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_child != mixed_result.m_tx_results.end()); - - BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent1->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent2_v1->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); - - // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. - const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); - BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); - BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_mixed.value()); + } else { + auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); + auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); + auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); + auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); + + BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); + + // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. + const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); + BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); + BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } } } @@ -715,15 +683,17 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(submit_cpfp_deprio.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - const CFeeRate expected_feerate(0, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp_deprio.value()); + } else { + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } } // Clear the prioritisation of the parent transaction. @@ -735,31 +705,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); + if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp.value()); + } else { + auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); + + const CFeeRate expected_feerate(coinbase_value - child_value, + GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_cpfp.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_cpfp.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_cpfp.m_tx_results.size(), package_cpfp.size()); - auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); - BOOST_CHECK(it_child != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); - - const CFeeRate expected_feerate(coinbase_value - child_value, - GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); } // Just because we allow low-fee parents doesn't mean we allow low-feerate packages. @@ -785,15 +751,28 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) package_still_too_low.push_back(tx_child_cheap); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_child_cheap)) <= child_fee); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)) > parent_fee + child_fee); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - // Cheap package should fail with package-fee-too-low. + // Cheap package should fail for being too low fee. { - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_package_too_low.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "package-fee-too-low"); + if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_package_too_low.value()); + } else { + // Individual feerate of parent is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee, GetVirtualTransactionSize(*tx_parent_cheap))); + // Package feerate of parent + child is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee + child_fee, GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap))); + } + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "transaction failed"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } @@ -804,25 +783,26 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); + if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_prioritised.value()); + } else { + const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); + auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); + auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_prioritised_package.m_state.IsValid(), - "Package validation unexpectedly failed" << submit_prioritised_package.m_state.GetRejectReason()); - const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, - GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); - BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); - auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); - auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } // Package feerate is calculated without topology in mind; it's just aggregating fees and sizes. @@ -851,31 +831,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_rich_parent, /*test_accept=*/false); + if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_rich_parent.value()); + } else { + // The child would have been validated on its own and failed. + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); + + auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); + auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); + BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, + strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); + BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); + BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); + } expected_pool_size += 1; - BOOST_CHECK_MESSAGE(submit_rich_parent.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - - // The child would have been validated on its own and failed. - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); - - auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); - auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); - BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, - strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); - BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent_rich->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_poor->GetHash()))); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index bf5a653090..e0404e33ed 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -4,11 +4,15 @@ #include <test/util/net.h> -#include <chainparams.h> -#include <node/eviction.h> #include <net.h> #include <net_processing.h> +#include <netaddress.h> #include <netmessagemaker.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <protocol.h> +#include <random.h> +#include <serialize.h> #include <span.h> #include <vector> @@ -98,6 +102,17 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) co return complete; } +CNode* ConnmanTestMsg::ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) +{ + CNode* node = ConnectNode(CAddress{}, pszDest, /*fCountFailure=*/false, conn_type, /*use_v2transport=*/true); + if (!node) return nullptr; + node->SetCommonVersion(PROTOCOL_VERSION); + peerman.InitializeNode(*node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node->fSuccessfullyConnected = true; + AddTestNode(*node); + return node; +} + std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) { std::vector<NodeEvictionCandidate> candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index 497292542b..59c4ddb4b1 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,16 +6,30 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat/compat.h> -#include <node/eviction.h> -#include <netaddress.h> #include <net.h> +#include <net_permissions.h> +#include <net_processing.h> +#include <netaddress.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <sync.h> #include <util/sock.h> +#include <algorithm> #include <array> #include <cassert> +#include <chrono> +#include <cstdint> #include <cstring> #include <memory> #include <string> +#include <unordered_map> +#include <vector> + +class FastRandomContext; + +template <typename C> +class Span; struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; @@ -25,6 +39,12 @@ struct ConnmanTestMsg : public CConnman { m_peer_connect_timeout = timeout; } + std::vector<CNode*> TestNodes() + { + LOCK(m_nodes_mutex); + return m_nodes; + } + void AddTestNode(CNode& node) { LOCK(m_nodes_mutex); @@ -56,6 +76,11 @@ struct ConnmanTestMsg : public CConnman { bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const; void FlushSendBuffer(CNode& node) const; + + bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); }; + + CNode* ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ diff --git a/src/test/util/script.h b/src/test/util/script.h index 428b3e10b3..96c4d55e5a 100644 --- a/src/test/util/script.h +++ b/src/test/util/script.h @@ -18,6 +18,18 @@ static const CScript P2WSH_OP_TRUE{ return hash; }())}; +static const std::vector<uint8_t> EMPTY{}; +static const CScript P2WSH_EMPTY{ + CScript{} + << OP_0 + << ToByteVector([] { + uint256 hash; + CSHA256().Write(EMPTY.data(), EMPTY.size()).Finalize(hash.begin()); + return hash; + }())}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TRUE_STACK{{static_cast<uint8_t>(OP_TRUE)}, {}}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TWO_STACK{{static_cast<uint8_t>(OP_2)}, {}}; + /** Flags that are not forbidden by an assert in script validation */ bool IsValidFlagCombination(unsigned flags); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 6ae94a8101..9979b75444 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -147,6 +147,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto BasicTestingSetup::~BasicTestingSetup() { + m_node.kernel.reset(); SetMockTime(0s); // Reset mocktime for following tests LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); @@ -205,8 +206,9 @@ ChainTestingSetup::~ChainTestingSetup() m_node.netgroupman.reset(); m_node.args = nullptr; m_node.mempool.reset(); - m_node.scheduler.reset(); + m_node.fee_estimator.reset(); m_node.chainman.reset(); + m_node.scheduler.reset(); } void ChainTestingSetup::LoadVerifyActivateChainstate() @@ -520,6 +522,6 @@ CBlock getBlock13b8a() { CBlock block; CDataStream stream(ParseHex("0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab24889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c311b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac000000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e571fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac000000000100000002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe67512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035ddefb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987dad92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a9145505614859643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acfcab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd2000000008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc6514edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229cefc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b63f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a33161dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a9059cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c353069e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce3012e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); return block; } diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 147e589deb..6d3bb01be8 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -89,11 +89,14 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, } // m_effective_feerate and m_wtxids_fee_calculations should exist iff the result was valid - if (atmp_result.m_effective_feerate.has_value() != valid) { + // or if the failure was TX_RECONSIDERABLE + const bool valid_or_reconsiderable{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID || + atmp_result.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + if (atmp_result.m_effective_feerate.has_value() != valid_or_reconsiderable) { return strprintf("tx %s result should %shave m_effective_feerate", wtxid.ToString(), valid ? "" : "not "); } - if (atmp_result.m_wtxids_fee_calculations.has_value() != valid) { + if (atmp_result.m_wtxids_fee_calculations.has_value() != valid_or_reconsiderable) { return strprintf("tx %s result should %shave m_effective_feerate", wtxid.ToString(), valid ? "" : "not "); } diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 64cb5522eb..35e5c6a037 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -283,8 +283,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Check that all txs are in the pool { - LOCK(m_node.mempool->cs); - BOOST_CHECK_EQUAL(m_node.mempool->mapTx.size(), txs.size()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), txs.size()); } // Run a thread that simulates an RPC caller that is polling while @@ -295,7 +294,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // not some intermediate amount. while (true) { LOCK(m_node.mempool->cs); - if (m_node.mempool->mapTx.size() == 0) { + if (m_node.mempool->size() == 0) { // We are done with the reorg break; } @@ -304,7 +303,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // be atomic. So the caller assumes that the returned mempool // is consistent. That is, it has all txs that were there // before the reorg. - assert(m_node.mempool->mapTx.size() == txs.size()); + assert(m_node.mempool->size() == txs.size()); continue; } LOCK(cs_main); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 461662ad93..e057d7ece1 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -412,6 +412,7 @@ CTxMemPool::CTxMemPool(const Options& opts) m_max_datacarrier_bytes{opts.max_datacarrier_bytes}, m_require_standard{opts.require_standard}, m_full_rbf{opts.full_rbf}, + m_persist_v1_dat{opts.persist_v1_dat}, m_limits{opts.limits} { } @@ -480,8 +481,8 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces minerPolicyEstimator->processTransaction(entry, validFeeEstimate); } - vTxHashes.emplace_back(tx.GetWitnessHash(), newit); - newit->vTxHashesIdx = vTxHashes.size() - 1; + txns_randomized.emplace_back(newit->GetSharedTx()); + newit->idx_randomized = txns_randomized.size() - 1; TRACE3(mempool, added, entry.GetTx().GetHash().data(), @@ -517,14 +518,16 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); - if (vTxHashes.size() > 1) { - vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); - vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; - vTxHashes.pop_back(); - if (vTxHashes.size() * 2 < vTxHashes.capacity()) - vTxHashes.shrink_to_fit(); + if (txns_randomized.size() > 1) { + // Update idx_randomized of the to-be-moved entry. + Assert(GetEntry(txns_randomized.back()->GetHash()))->idx_randomized = it->idx_randomized; + // Remove entry from txns_randomized by replacing it with the back and deleting the back. + txns_randomized[it->idx_randomized] = std::move(txns_randomized.back()); + txns_randomized.pop_back(); + if (txns_randomized.size() * 2 < txns_randomized.capacity()) + txns_randomized.shrink_to_fit(); } else - vTxHashes.clear(); + txns_randomized.clear(); totalTxSize -= it->GetTxSize(); m_total_fee -= it->GetFee(); @@ -836,6 +839,18 @@ static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), it->GetFee(), it->GetTxSize(), it->GetModifiedFee() - it->GetFee()}; } +std::vector<CTxMemPoolEntryRef> CTxMemPool::entryAll() const +{ + AssertLockHeld(cs); + + std::vector<CTxMemPoolEntryRef> ret; + ret.reserve(mapTx.size()); + for (const auto& it : GetSortedDepthAndScore()) { + ret.emplace_back(*it); + } + return ret; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -850,6 +865,13 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const return ret; } +const CTxMemPoolEntry* CTxMemPool::GetEntry(const Txid& txid) const +{ + AssertLockHeld(cs); + const auto i = mapTx.find(txid); + return i == mapTx.end() ? nullptr : &(*i); +} + CTransactionRef CTxMemPool::get(const uint256& hash) const { LOCK(cs); @@ -1033,7 +1055,7 @@ void CCoinsViewMemPool::Reset() size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(txns_randomized) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { diff --git a/src/txmempool.h b/src/txmempool.h index 374c212a44..aed6acd9da 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -392,7 +392,7 @@ public: indexed_transaction_set mapTx GUARDED_BY(cs); using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; - std::vector<std::pair<uint256, txiter>> vTxHashes GUARDED_BY(cs); //!< All tx witness hashes/entries in mapTx, in random order + std::vector<CTransactionRef> txns_randomized GUARDED_BY(cs); //!< All transactions in mapTx, in random order typedef std::set<txiter, CompareIteratorByHash> setEntries; @@ -446,6 +446,7 @@ public: const std::optional<unsigned> m_max_datacarrier_bytes; const bool m_require_standard; const bool m_full_rbf; + const bool m_persist_v1_dat; const Limits m_limits; @@ -684,6 +685,8 @@ public: return (mapTx.count(gtxid.GetHash()) != 0); } + const CTxMemPoolEntry* GetEntry(const Txid& txid) const LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(cs); + CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) { @@ -695,6 +698,7 @@ public: /** Returns info for a transaction if its entry_sequence < last_sequence */ TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<CTxMemPoolEntryRef> entryAll() const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/util/fs.h b/src/util/fs.h index 8f79f6cba6..7e2803b6aa 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -184,6 +184,7 @@ static inline path PathFromString(const std::string& string) * already exists or is a symlink to an existing directory. * This is a temporary workaround for an issue in libstdc++ that has been fixed * upstream [PR101510]. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510 */ static inline bool create_directories(const std::filesystem::path& p) { diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index 2a9eb3502e..8aa7493aa8 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -11,13 +11,11 @@ #include <logging.h> #include <sync.h> -#include <tinyformat.h> #include <util/fs.h> #include <util/getuniquepath.h> #include <util/syserror.h> #include <cerrno> -#include <filesystem> #include <fstream> #include <map> #include <memory> @@ -263,7 +261,7 @@ bool RenameOver(fs::path src, fs::path dest) { #ifdef __MINGW64__ // This is a workaround for a bug in libstdc++ which - // implements std::filesystem::rename with _wrename function. + // implements fs::rename with _wrename function. // This bug has been fixed in upstream: // - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e // - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312 diff --git a/src/util/sock.cpp b/src/util/sock.cpp index d16dc56aa3..e896b87160 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -242,7 +242,7 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per #endif /* USE_POLL */ } -void Sock::SendComplete(const std::string& data, +void Sock::SendComplete(Span<const unsigned char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const { @@ -283,6 +283,13 @@ void Sock::SendComplete(const std::string& data, } } +void Sock::SendComplete(Span<const char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + SendComplete(MakeUCharSpan(data), timeout, interrupt); +} + std::string Sock::RecvUntilTerminator(uint8_t terminator, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt, diff --git a/src/util/sock.h b/src/util/sock.h index d78e01929b..65e7ffc165 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -228,7 +228,14 @@ public: * @throws std::runtime_error if the operation cannot be completed. In this case only some of * the data will be written to the socket. */ - virtual void SendComplete(const std::string& data, + virtual void SendComplete(Span<const unsigned char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Convenience method, equivalent to `SendComplete(MakeUCharSpan(data), timeout, interrupt)`. + */ + virtual void SendComplete(Span<const char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const; diff --git a/src/validation.cpp b/src/validation.cpp index 8d7e366125..ed72a7c97a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -353,7 +353,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( const std::optional<LockPoints> new_lock_points{CalculateLockPointsAtTip(m_chain.Tip(), view_mempool, tx)}; if (new_lock_points.has_value() && CheckSequenceLocksAtTip(m_chain.Tip(), *new_lock_points)) { // Now update the mempool entry lockpoints as well. - m_mempool->mapTx.modify(it, [&new_lock_points](CTxMemPoolEntry& e) { e.UpdateLockPoints(*new_lock_points); }); + it->UpdateLockPoints(*new_lock_points); } else { return true; } @@ -362,9 +362,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( // If the transaction spends any coinbase outputs, it must be mature. if (it->GetSpendsCoinbase()) { for (const CTxIn& txin : tx.vin) { - auto it2 = m_mempool->mapTx.find(txin.prevout.hash); - if (it2 != m_mempool->mapTx.end()) - continue; + if (m_mempool->exists(GenTxid::Txid(txin.prevout.hash))) continue; const Coin& coin{CoinsTip().AccessCoin(txin.prevout)}; assert(!coin.IsSpent()); const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; @@ -670,11 +668,11 @@ private: AssertLockHeld(m_pool.cs); CAmount mempoolRejectFee = m_pool.GetMinFee().GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < m_pool.m_min_relay_feerate.GetFee(package_size)) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "min relay fee not met", strprintf("%d < %d", package_fee, m_pool.m_min_relay_feerate.GetFee(package_size))); } return true; @@ -724,7 +722,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } // Transactions smaller than 65 non-witness bytes are not relayed to mitigate CVE-2017-12842. - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) < MIN_STANDARD_TX_NONWITNESS_SIZE) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next @@ -867,6 +865,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. if (!bypass_limits && ws.m_modified_fees < m_pool.m_min_relay_feerate.GetFee(ws.m_vsize)) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", ws.m_modified_fees, m_pool.m_min_relay_feerate.GetFee(ws.m_vsize))); } @@ -981,6 +981,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } @@ -1002,6 +1005,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) } if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_incremental_relay_feerate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; @@ -1139,7 +1145,8 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) if (!args.m_package_submission && !bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); if (!m_pool.exists(GenTxid::Txid(hash))) - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); + // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); } return true; } @@ -1201,7 +1208,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& } } - std::vector<uint256> all_package_wtxids; + std::vector<Wtxid> all_package_wtxids; all_package_wtxids.reserve(workspaces.size()); std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); @@ -1211,7 +1218,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>({ws.m_ptx->GetWitnessHash()}); + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); @@ -1226,8 +1233,15 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) Workspace ws(ptx); + const std::vector<Wtxid> single_wtxid{ws.m_ptx->GetWitnessHash()}; - if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!PreChecks(args, ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); @@ -1238,14 +1252,18 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); const CFeeRate effective_feerate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; - const std::vector<uint256> single_wtxid{ws.m_ptx->GetWitnessHash()}; // Tx was accepted, but not added if (args.m_test_accept) { return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } - if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!Finalize(args, ws)) { + // The only possible failure reason is fee-related (mempool full). + // Failed for fee reasons. Provide the effective feerate and which txns were included. + Assume(ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE); + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + } GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); @@ -1299,11 +1317,16 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + std::vector<Wtxid> all_package_wtxids; + all_package_wtxids.reserve(workspaces.size()); + std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), + [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); TxValidationState placeholder_state; if (args.m_package_feerates && !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-fee-too-low"); - return PackageMempoolAcceptResult(package_state, {}); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + return PackageMempoolAcceptResult(package_state, {{workspaces.back().m_ptx->GetWitnessHash(), + MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_total_modified_fees, m_total_vsize), all_package_wtxids)}}); } // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, @@ -1314,10 +1337,6 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } - std::vector<uint256> all_package_wtxids; - all_package_wtxids.reserve(workspaces.size()); - std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), - [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); for (Workspace& ws : workspaces) { ws.m_package_feerate = package_feerate; if (!PolicyScriptChecks(args, ws)) { @@ -1330,7 +1349,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>{ws.m_ptx->GetWitnessHash()}; + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, @@ -1485,9 +1504,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with // the new transactions. This ensures we don't double-count transaction counts and sizes when // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(iter.value()->GetTxSize(), iter.value()->GetFee())); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(entry.GetTxSize(), entry.GetFee())); } else if (m_pool.exists(GenTxid::Txid(txid))) { // Transaction with the same non-witness data but different witness (same txid, // different wtxid) already exists in the mempool. @@ -1496,10 +1514,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transaction for the mempool one. Note that we are ignoring the validity of the // package transaction passed in. // TODO: allow witness replacement in packages. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; // Provide the wtxid of the mempool tx so that the caller can look it up in the mempool. - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(entry.GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. // Try submitting the transaction on its own. @@ -1510,7 +1527,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // in package validation, because its fees should only be "used" once. assert(m_pool.exists(GenTxid::Wtxid(wtxid))); results_final.emplace(wtxid, single_res); - } else if (single_res.m_state.GetResult() != TxValidationResult::TX_MEMPOOL_POLICY && + } else if (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { // Package validation policy only differs from individual policy in its evaluation // of feerate. For example, if a transaction fails here due to violation of a @@ -3668,7 +3685,7 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(TX_NO_WITNESS(block)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be @@ -4707,7 +4724,7 @@ void ChainstateManager::LoadExternalBlockFile( // This block can be processed immediately; rewind to its start, read and deserialize it. blkdat.SetPos(nBlockPos); pblock = std::make_shared<CBlock>(); - blkdat >> *pblock; + blkdat >> TX_WITH_WITNESS(*pblock); nRewind = blkdat.GetPos(); BlockValidationState state; diff --git a/src/validation.h b/src/validation.h index 7ce60da634..e669ec46d5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -114,7 +114,27 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); /** -* Validation result for a single transaction mempool acceptance. +* Validation result for a transaction evaluated by MemPoolAccept (single or package). +* Here are the expected fields and properties of a result depending on its ResultType, applicable to +* results returned from package evaluation: +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| Field or property | VALID | INVALID | MEMPOOL_ENTRY | DIFFERENT_WITNESS | +*| | |--------------------------------------| | | +*| | | TX_RECONSIDERABLE | Other | | | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| txid in mempool? | yes | no | no* | yes | yes | +*| wtxid in mempool? | yes | no | no* | yes | no | +*| m_state | yes, IsValid() | yes, IsInvalid() | yes, IsInvalid() | yes, IsValid() | yes, IsValid() | +*| m_replaced_transactions | yes | no | no | no | no | +*| m_vsize | yes | no | no | yes | no | +*| m_base_fees | yes | no | no | yes | no | +*| m_effective_feerate | yes | yes | no | no | no | +*| m_wtxids_fee_calculations | yes | yes | no | no | no | +*| m_other_wtxid | no | no | no | no | yes | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +* (*) Individual transaction acceptance doesn't return MEMPOOL_ENTRY and DIFFERENT_WITNESS. It returns +* INVALID, with the errors txn-already-in-mempool and txn-same-nonwitness-data-in-mempool +* respectively. In those cases, the txid or wtxid may be in the mempool for a TX_CONFLICT. */ struct MempoolAcceptResult { /** Used to indicate the results of mempool validation. */ @@ -130,7 +150,6 @@ struct MempoolAcceptResult { /** Contains information about why the transaction failed. */ const TxValidationState m_state; - // The following fields are only present when m_result_type = ResultType::VALID or MEMPOOL_ENTRY /** Mempool transactions replaced by the tx. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ @@ -141,7 +160,6 @@ struct MempoolAcceptResult { * using prioritisetransaction (i.e. modified fees). If this transaction was submitted as a * package, this is the package feerate, which may also include its descendants and/or * ancestors (see m_wtxids_fee_calculations below). - * Only present when m_result_type = ResultType::VALID. */ const std::optional<CFeeRate> m_effective_feerate; /** Contains the wtxids of the transactions used for fee-related checks. Includes this @@ -149,9 +167,8 @@ struct MempoolAcceptResult { * package. This is not necessarily equivalent to the list of transactions passed to * ProcessNewPackage(). * Only present when m_result_type = ResultType::VALID. */ - const std::optional<std::vector<uint256>> m_wtxids_fee_calculations; + const std::optional<std::vector<Wtxid>> m_wtxids_fee_calculations; - // The following field is only present when m_result_type = ResultType::DIFFERENT_WITNESS /** The wtxid of the transaction in the mempool which has the same txid but different witness. */ const std::optional<uint256> m_other_wtxid; @@ -159,11 +176,17 @@ struct MempoolAcceptResult { return MempoolAcceptResult(state); } + static MempoolAcceptResult FeeFailure(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) { + return MempoolAcceptResult(state, effective_feerate, wtxids_fee_calculations); + } + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) { + const std::vector<Wtxid>& wtxids_fee_calculations) { return MempoolAcceptResult(std::move(replaced_txns), vsize, fees, effective_feerate, wtxids_fee_calculations); } @@ -189,7 +212,7 @@ private: int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) + const std::vector<Wtxid>& wtxids_fee_calculations) : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, @@ -197,6 +220,15 @@ private: m_effective_feerate(effective_feerate), m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for fee-related failure case */ + explicit MempoolAcceptResult(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) + : m_result_type(ResultType::INVALID), + m_state(state), + m_effective_feerate(effective_feerate), + m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for already-in-mempool case. It wouldn't replace any transactions. */ explicit MempoolAcceptResult(int64_t vsize, CAmount fees) : m_result_type(ResultType::MEMPOOL_ENTRY), m_vsize{vsize}, m_base_fees(fees) {} diff --git a/src/version.h b/src/version.h index 204df3758b..c2ebeecbfb 100644 --- a/src/version.h +++ b/src/version.h @@ -35,7 +35,4 @@ static const int INVALID_CB_NO_BAN_VERSION = 70015; //! "wtxidrelay" command for wtxid-based relay starts with this version static const int WTXID_RELAY_VERSION = 70016; -// Make sure that none of the values above collide with -// `SERIALIZE_TRANSACTION_NO_WITNESS`. - #endif // BITCOIN_VERSION_H diff --git a/src/wallet/context.h b/src/wallet/context.h index 57a6ed77f7..58b9924fb4 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -13,6 +13,7 @@ #include <vector> class ArgsManager; +class CScheduler; namespace interfaces { class Chain; class Wallet; @@ -34,6 +35,7 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wall //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + CScheduler* scheduler{nullptr}; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct // It is unsafe to lock this after locking a CWallet::cs_wallet mutex because // this could introduce inconsistent lock ordering and cause deadlocks. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index cafa6c70a1..d9a08310a8 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -86,7 +86,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans reused_inputs.push_back(txin.prevout); } - std::optional<CAmount> combined_bump_fee = wallet.chain().CalculateCombinedBumpFee(reused_inputs, newFeerate); + std::optional<CAmount> combined_bump_fee = wallet.chain().calculateCombinedBumpFee(reused_inputs, newFeerate); if (!combined_bump_fee.has_value()) { errors.push_back(strprintf(Untranslated("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions."))); } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 0d0a8650ac..088343458c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -95,8 +95,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-walletcrosschain", strprintf("Allow reusing wallet files across chains (default: %u)", DEFAULT_WALLETCROSSCHAIN), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - - argsman.AddHiddenArgs({"-zapwallettxes"}); } bool WalletInit::ParameterInteraction() const @@ -118,10 +116,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.IsArgSet("-zapwallettxes")) { - return InitError(Untranslated("-zapwallettxes has been removed. If you are attempting to remove a stuck transaction from your wallet, please use abandontransaction instead.")); - } - return true; } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index c9419be0ab..4ca5e29229 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -11,6 +11,7 @@ #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> +#include <scheduler.h> #include <support/allocators/secure.h> #include <sync.h> #include <uint256.h> @@ -212,7 +213,7 @@ public: } return true; } - std::vector<WalletAddress> getAddresses() const override + std::vector<WalletAddress> getAddresses() override { LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; @@ -585,10 +586,15 @@ public: } bool verify() override { return VerifyWallets(m_context); } bool load() override { return LoadWallets(m_context); } - void start(CScheduler& scheduler) override { return StartWallets(m_context, scheduler); } + void start(CScheduler& scheduler) override + { + m_context.scheduler = &scheduler; + return StartWallets(m_context); + } void flush() override { return FlushWallets(m_context); } void stop() override { return StopWallets(m_context); } void setMockTime(int64_t time) override { return SetMockTime(time); } + void schedulerMockForward(std::chrono::seconds delta) override { Assert(m_context.scheduler)->MockForward(delta); } //! WalletLoader methods util::Result<std::unique_ptr<Wallet>> createWallet(const std::string& name, const SecureString& passphrase, uint64_t wallet_creation_flags, std::vector<bilingual_str>& warnings) override diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 4cdfadbee2..8b78a670e4 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -141,7 +141,7 @@ bool LoadWallets(WalletContext& context) } } -void StartWallets(WalletContext& context, CScheduler& scheduler) +void StartWallets(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->postInitProcess(); @@ -149,9 +149,9 @@ void StartWallets(WalletContext& context, CScheduler& scheduler) // Schedule periodic wallet flushes and tx rebroadcasts if (context.args->GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - scheduler.scheduleEvery([&context] { MaybeCompactWalletDB(context); }, std::chrono::milliseconds{500}); + context.scheduler->scheduleEvery([&context] { MaybeCompactWalletDB(context); }, 500ms); } - scheduler.scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); + context.scheduler->scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); } void FlushWallets(WalletContext& context) diff --git a/src/wallet/load.h b/src/wallet/load.h index 0882f7f8ad..c079cad955 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -26,7 +26,7 @@ bool VerifyWallets(WalletContext& context); bool LoadWallets(WalletContext& context); //! Complete startup of wallets. -void StartWallets(WalletContext& context, CScheduler& scheduler); +void StartWallets(WalletContext& context); //! Flush all wallets in preparation for shutdown. void FlushWallets(WalletContext& context); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 6b96fc4e49..0f7a882034 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1617,8 +1617,8 @@ RPCHelpMan walletprocesspsbt() CMutableTransaction mtx; // Returns true if complete, which we already think it is. CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index c34391e6e8..cd7ed461f0 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -783,7 +783,7 @@ RPCHelpMan gettransaction() ListTransactions(*pwallet, wtx, 0, false, details, filter, /*filter_label=*/std::nullopt); entry.pushKV("details", details); - std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); + std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationWithoutWitness()); entry.pushKV("hex", strHex); if (verbose) { diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index ed621cad61..b573062d32 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -259,7 +259,7 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const { PreSelectedInputs result; const bool can_grind_r = wallet.CanGrindR(); - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); for (const COutPoint& outpoint : coin_control.ListSelected()) { int input_bytes = -1; CTxOut txout; @@ -453,7 +453,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, } if (feerate.has_value()) { - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(outpoints, feerate.value()); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(outpoints, feerate.value()); for (auto& [_, outputs] : result.coins) { for (auto& output : outputs) { @@ -725,7 +725,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co outpoints.push_back(coin->outpoint); summed_bump_fees += coin->ancestor_bump_fees; } - std::optional<CAmount> combined_bump_fee = chain.CalculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); + std::optional<CAmount> combined_bump_fee = chain.calculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); if (!combined_bump_fee.has_value()) { return util::Error{_("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions.")}; } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9510f28282..b46f361363 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -34,12 +34,12 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx1; - s_prev_tx1 >> prev_tx1; + s_prev_tx1 >> TX_WITH_WITNESS(prev_tx1); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1, TxStateInactive{})); CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx2; - s_prev_tx2 >> prev_tx2; + s_prev_tx2 >> TX_WITH_WITNESS(prev_tx2); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2, TxStateInactive{})); // Import descriptors for keys and scripts diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 4f78fe7520..6777257e53 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -4,6 +4,10 @@ #include <wallet/transaction.h> +#include <interfaces/chain.h> + +using interfaces::FoundBlock; + namespace wallet { bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { @@ -25,6 +29,27 @@ int64_t CWalletTx::GetTxTime() const return n ? n : nTimeReceived; } +void CWalletTx::updateState(interfaces::Chain& chain) +{ + bool active; + auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. + if (!chain.findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { + state = TxStateInactive{}; + } + }; + if (auto* conf = state<TxStateConfirmed>()) { + lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); + } else if (auto* conf = state<TxStateConflicted>()) { + lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); + } +} + void CWalletTx::CopyFrom(const CWalletTx& _tx) { *this = _tx; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index db858fa5ba..ddeb931112 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -22,6 +22,10 @@ #include <variant> #include <vector> +namespace interfaces { +class Chain; +} // namespace interfaces + namespace wallet { //! State of transaction confirmed in a block. struct TxStateConfirmed { @@ -161,7 +165,7 @@ public: std::vector<uint256> vMerkleBranch; int nIndex; - s >> tx >> hashBlock >> vMerkleBranch >> nIndex; + s >> TX_WITH_WITNESS(tx) >> hashBlock >> vMerkleBranch >> nIndex; } }; @@ -272,7 +276,7 @@ public: bool dummy_bool = false; //!< Used to be fSpent uint256 serializedHash = TxStateSerializedBlockHash(m_state); int serializedIndex = TxStateSerializedIndex(m_state); - s << tx << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; + s << TX_WITH_WITNESS(tx) << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; } template<typename Stream> @@ -285,7 +289,7 @@ public: bool dummy_bool; //! Used to be fSpent uint256 serialized_block_hash; int serializedIndex; - s >> tx >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; + s >> TX_WITH_WITNESS(tx) >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; m_state = TxStateInterpretSerialized({serialized_block_hash, serializedIndex}); @@ -326,6 +330,10 @@ public: template<typename T> const T* state() const { return std::get_if<T>(&m_state); } template<typename T> T* state() { return std::get_if<T>(&m_state); } + //! Update transaction state when attaching to a chain, filling in heights + //! of conflicted and confirmed blocks + void updateState(interfaces::Chain& chain); + bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } bool isConflicted() const { return state<TxStateConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 162d7f9ec7..ecf18fbe78 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1184,23 +1184,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx // If wallet doesn't have a chain (e.g when using bitcoin-wallet tool), // don't bother to update txn. if (HaveChain()) { - bool active; - auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { - // If tx block (or conflicting block) was reorged out of chain - // while the wallet was shutdown, change tx status to UNCONFIRMED - // and reset block height, hash, and index. ABANDONED tx don't have - // associated blocks and don't need to be updated. The case where a - // transaction was reorged out while online and then reconfirmed - // while offline is covered by the rescan logic. - if (!chain().findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { - state = TxStateInactive{}; - } - }; - if (auto* conf = wtx.state<TxStateConfirmed>()) { - lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, wtx.m_state); - } else if (auto* conf = wtx.state<TxStateConflicted>()) { - lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, wtx.m_state); - } + wtx.updateState(chain()); } if (/* insertion took place */ ins.second) { wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -3319,8 +3303,10 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { AssertLockHeld(cs_wallet); if (auto* conf = wtx.state<TxStateConfirmed>()) { + assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; } else if (auto* conf = wtx.state<TxStateConflicted>()) { + assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { return 0; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 85c91ef15a..bc45010200 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -503,6 +503,13 @@ public: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain + * + * Preconditions: it is only valid to call this function when the wallet is + * online and the block index is loaded. So this cannot be called by + * bitcoin-wallet tool code or by wallet migration code. If this is called + * without the wallet being online, it won't be able able to determine the + * the height of the last block processed, or the heights of blocks + * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 1241431523..a4fb915d71 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -244,14 +244,14 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + DataStream ss; CBlock block; if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } - ss << block; + ss << RPCTxSerParams(block); return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } @@ -260,8 +260,8 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "Publish rawtx %s to %s\n", hash.GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ss << transaction; + DataStream ss; + ss << RPCTxSerParams(transaction); return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 5e9c4198f1..2e3589b020 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -118,8 +118,14 @@ class AssumeutxoTest(BitcoinTestFramework): chainstate_snapshot_path.mkdir() with open(chainstate_snapshot_path / "base_blockhash", 'wb') as f: f.write(b'z' * 32) - expected_error = f"Error: A fatal internal error occurred, see debug.log for details" - self.nodes[0].assert_start_raises_init_error(expected_msg=expected_error) + + def expected_error(log_msg="", error_msg=""): + with self.nodes[0].assert_debug_log([log_msg]): + self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) + + expected_error_msg = f"Error: A fatal internal error occurred, see debug.log for details" + error_details = f"Assumeutxo data not found for the given blockhash" + expected_error(log_msg=error_details, error_msg=expected_error_msg) # resurrect node again rmtree(chainstate_snapshot_path) diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 37ef3de4dd..142d75a851 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2021-2022 The Bitcoin Core developers +# Copyright (c) 2021-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Stress tests related to node initialization.""" import os from pathlib import Path -from random import randint import shutil from test_framework.test_framework import BitcoinTestFramework, SkipTest @@ -138,8 +137,8 @@ class InitStressTest(BitcoinTestFramework): # Since the genesis block is not checked by -checkblocks, the # perturbation window must be chosen such that a higher block # in blk*.dat is affected. - tf.seek(randint (150, 15000)) - tf.write(b'1' * randint(20, 2000)) + tf.seek(150) + tf.write(b"1" * 200) start_expecting_error(err_fragment) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 1d8c87feb5..e85541d0ec 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -104,7 +104,7 @@ from test_framework.key import ( tweak_add_privkey, ECKey, ) -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 from test_framework.address import ( hash160, program_to_witness, diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index ce2a5ab8ac..be154b411f 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -11,7 +11,7 @@ from test_framework.messages import ( COutPoint, from_hex, ) -from test_framework.muhash import MuHash3072 +from test_framework.crypto.muhash import MuHash3072 from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.wallet import MiniWallet diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index fd3e219586..a126f164aa 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -28,7 +28,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): def setup_network(self): self.add_nodes(self.num_nodes, versions=[ - 200100, # Last release with previous mempool format + 200100, # Last release without unbroadcast serialization and without XOR None, ]) self.start_nodes() @@ -59,7 +59,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): old_node_mempool.rename(new_node_mempool) self.log.info("Start new node and verify mempool contains the tx") - self.start_node(1) + self.start_node(1, extra_args=["-persistmempoolv1=1"]) assert old_tx_hash in new_node.getrawmempool() self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") diff --git a/test/functional/p2p_v2_transport.py b/test/functional/p2p_v2_transport.py index 5ad2194b84..1a3b4a6d0a 100755 --- a/test/functional/p2p_v2_transport.py +++ b/test/functional/p2p_v2_transport.py @@ -48,7 +48,7 @@ class V2TransportTest(BitcoinTestFramework): assert_equal(self.nodes[1].getblockcount(), 5) # verify there is a v2 connection between node 0 and 1 node_0_info = self.nodes[0].getpeerinfo() - node_1_info = self.nodes[0].getpeerinfo() + node_1_info = self.nodes[1].getpeerinfo() assert_equal(len(node_0_info), 1) assert_equal(len(node_1_info), 1) assert_equal(node_0_info[0]["transport_protocol_type"], "v2") diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 50a022fc7e..773ab3b50e 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -215,8 +215,11 @@ class NetTest(BitcoinTestFramework): # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) self.nodes[0].addnode(node=ip_port, command='add') + # try to add an equivalent ip + ip_port2 = "127.1:{}".format(p2p_port(2)) + assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port2, command='add') # check that the node has indeed been added - added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + added_nodes = self.nodes[0].getaddednodeinfo() assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) # check that node cannot be added again diff --git a/test/functional/test_framework/blockfilter.py b/test/functional/test_framework/blockfilter.py index 17d81a4cb5..a16aa3d34f 100644 --- a/test/functional/test_framework/blockfilter.py +++ b/test/functional/test_framework/blockfilter.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helper routines relevant for compact block filters (BIP158). """ -from .siphash import siphash +from .crypto.siphash import siphash def bip158_basic_element_hash(script_pub_key, N, block_hash): diff --git a/test/functional/test_framework/crypto/bip324_cipher.py b/test/functional/test_framework/crypto/bip324_cipher.py new file mode 100644 index 0000000000..56190647f2 --- /dev/null +++ b/test/functional/test_framework/crypto/bip324_cipher.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 Poly1305 AEAD Construction in RFC 8439 and FSChaCha20Poly1305 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +from .chacha20 import chacha20_block, REKEY_INTERVAL +from .poly1305 import Poly1305 + + +def pad16(x): + if len(x) % 16 == 0: + return b'' + return b'\x00' * (16 - (len(x) % 16)) + + +def aead_chacha20_poly1305_encrypt(key, nonce, aad, plaintext): + """Encrypt a plaintext using ChaCha20Poly1305.""" + ret = bytearray() + msg_len = len(plaintext) + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(plaintext[j + 64 * i] ^ keystream[j]) + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ret + pad16(ret) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + ret += poly1305.tag(mac_data) + return bytes(ret) + + +def aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext): + """Decrypt a ChaCha20Poly1305 ciphertext.""" + if len(ciphertext) < 16: + return None + msg_len = len(ciphertext) - 16 + poly1305 = Poly1305(chacha20_block(key, nonce, 0)[:32]) + mac_data = aad + pad16(aad) + mac_data += ciphertext[:-16] + pad16(ciphertext[:-16]) + mac_data += len(aad).to_bytes(8, 'little') + msg_len.to_bytes(8, 'little') + if ciphertext[-16:] != poly1305.tag(mac_data): + return None + ret = bytearray() + for i in range((msg_len + 63) // 64): + now = min(64, msg_len - 64 * i) + keystream = chacha20_block(key, nonce, i + 1) + for j in range(now): + ret.append(ciphertext[j + 64 * i] ^ keystream[j]) + return bytes(ret) + + +class FSChaCha20Poly1305: + """Rekeying wrapper AEAD around ChaCha20Poly1305.""" + def __init__(self, initial_key): + self._key = initial_key + self._packet_counter = 0 + + def _crypt(self, aad, text, is_decrypt): + nonce = ((self._packet_counter % REKEY_INTERVAL).to_bytes(4, 'little') + + (self._packet_counter // REKEY_INTERVAL).to_bytes(8, 'little')) + if is_decrypt: + ret = aead_chacha20_poly1305_decrypt(self._key, nonce, aad, text) + else: + ret = aead_chacha20_poly1305_encrypt(self._key, nonce, aad, text) + if (self._packet_counter + 1) % REKEY_INTERVAL == 0: + rekey_nonce = b"\xFF\xFF\xFF\xFF" + nonce[4:] + self._key = aead_chacha20_poly1305_encrypt(self._key, rekey_nonce, b"", b"\x00" * 32)[:32] + self._packet_counter += 1 + return ret + + def decrypt(self, aad, ciphertext): + return self._crypt(aad, ciphertext, True) + + def encrypt(self, aad, plaintext): + return self._crypt(aad, plaintext, False) + + +# Test vectors from RFC8439 consisting of plaintext, aad, 32 byte key, 12 byte nonce and ciphertext +AEAD_TESTS = [ + # RFC 8439 Example from section 2.8.2 + ["4c616469657320616e642047656e746c656d656e206f662074686520636c6173" + "73206f66202739393a204966204920636f756c64206f6666657220796f75206f" + "6e6c79206f6e652074697020666f7220746865206675747572652c2073756e73" + "637265656e20776f756c642062652069742e", + "50515253c0c1c2c3c4c5c6c7", + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f", + [7, 0x4746454443424140], + "d31a8d34648e60db7b86afbc53ef7ec2a4aded51296e08fea9e2b5a736ee62d6" + "3dbea45e8ca9671282fafb69da92728b1a71de0a9e060b2905d6a5b67ecd3b36" + "92ddbd7f2d778b8c9803aee328091b58fab324e4fad675945585808b4831d7bc" + "3ff4def08e4b7a9de576d26586cec64b61161ae10b594f09e26a7e902ecbd060" + "0691"], + # RFC 8439 Test vector A.5 + ["496e7465726e65742d4472616674732061726520647261667420646f63756d65" + "6e74732076616c696420666f722061206d6178696d756d206f6620736978206d" + "6f6e74687320616e64206d617920626520757064617465642c207265706c6163" + "65642c206f72206f62736f6c65746564206279206f7468657220646f63756d65" + "6e747320617420616e792074696d652e20497420697320696e617070726f7072" + "6961746520746f2075736520496e7465726e65742d4472616674732061732072" + "65666572656e6365206d6174657269616c206f7220746f206369746520746865" + "6d206f74686572207468616e206173202fe2809c776f726b20696e2070726f67" + "726573732e2fe2809d", + "f33388860000000000004e91", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + [0, 0x0807060504030201], + "64a0861575861af460f062c79be643bd5e805cfd345cf389f108670ac76c8cb2" + "4c6cfc18755d43eea09ee94e382d26b0bdb7b73c321b0100d4f03b7f355894cf" + "332f830e710b97ce98c8a84abd0b948114ad176e008d33bd60f982b1ff37c855" + "9797a06ef4f0ef61c186324e2b3506383606907b6a7c02b0f9f6157b53c867e4" + "b9166c767b804d46a59b5216cde7a4e99040c5a40433225ee282a1b0a06c523e" + "af4534d7f83fa1155b0047718cbc546a0d072b04b3564eea1b422273f548271a" + "0bb2316053fa76991955ebd63159434ecebb4e466dae5a1073a6727627097a10" + "49e617d91d361094fa68f0ff77987130305beaba2eda04df997b714d6c6f2c29" + "a6ad5cb4022b02709beead9d67890cbb22392336fea1851f38"], + # Test vectors exercising aad and plaintext which are multiples of 16 bytes. + ["8d2d6a8befd9716fab35819eaac83b33269afb9f1a00fddf66095a6c0cd91951" + "a6b7ad3db580be0674c3f0b55f618e34", + "", + "72ddc73f07101282bbbcf853b9012a9f9695fc5d36b303a97fd0845d0314e0c3", + [0x3432b75f, 0xb3585537eb7f4024], + "f760b8224fb2a317b1b07875092606131232a5b86ae142df5df1c846a7f6341a" + "f2564483dd77f836be45e6230808ffe402a6f0a3e8be074b3d1f4ea8a7b09451"], + ["", + "36970d8a704c065de16250c18033de5a400520ac1b5842b24551e5823a3314f3" + "946285171e04a81ebfbe3566e312e74ab80e94c7dd2ff4e10de0098a58d0f503", + "77adda51d6730b9ad6c995658cbd49f581b2547e7c0c08fcc24ceec797461021", + [0x1f90da88, 0x75dafa3ef84471a4], + "aaae5bb81e8407c94b2ae86ae0c7efbe"], +] + +FSAEAD_TESTS = [ + ["d6a4cb04ef0f7c09c1866ed29dc24d820e75b0491032a51b4c3366f9ca35c19e" + "a3047ec6be9d45f9637b63e1cf9eb4c2523a5aab7b851ebeba87199db0e839cf" + "0d5c25e50168306377aedbe9089fd2463ded88b83211cf51b73b150608cc7a60" + "0d0f11b9a742948482e1b109d8faf15b450aa7322e892fa2208c6691e3fecf4c" + "711191b14d75a72147", + "786cb9b6ebf44288974cf0", + "5c9e1c3951a74fba66708bf9d2c217571684556b6a6a3573bff2847d38612654", + 500, + "9dcebbd3281ea3dd8e9a1ef7d55a97abd6743e56ebc0c190cb2c4e14160b385e" + "0bf508dddf754bd02c7c208447c131ce23e47a4a14dfaf5dd8bc601323950f75" + "4e05d46e9232f83fc5120fbbef6f5347a826ec79a93820718d4ec7a2b7cfaaa4" + "4b21e16d726448b62f803811aff4f6d827ed78e738ce8a507b81a8ae13131192" + "8039213de18a5120dc9b7370baca878f50ff254418de3da50c"], + ["8349b7a2690b63d01204800c288ff1138a1d473c832c90ea8b3fc102d0bb3adc" + "44261b247c7c3d6760bfbe979d061c305f46d94c0582ac3099f0bf249f8cb234", + "", + "3bd2093fcbcb0d034d8c569583c5425c1a53171ea299f8cc3bbf9ae3530adfce", + 60000, + "30a6757ff8439b975363f166a0fa0e36722ab35936abd704297948f45083f4d4" + "99433137ce931f7fca28a0acd3bc30f57b550acbc21cbd45bbef0739d9caf30c" + "14b94829deb27f0b1923a2af704ae5d6"], +] + + +class TestFrameworkAEAD(unittest.TestCase): + def test_aead(self): + """ChaCha20Poly1305 AEAD test vectors.""" + for test_vector in AEAD_TESTS: + hex_plain, hex_aad, hex_key, hex_nonce, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + nonce = hex_nonce[0].to_bytes(4, 'little') + hex_nonce[1].to_bytes(8, 'little') + + ciphertext = aead_chacha20_poly1305_encrypt(key, nonce, aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + plaintext = aead_chacha20_poly1305_decrypt(key, nonce, aad, ciphertext) + self.assertEqual(plain, plaintext) + + def test_fschacha20poly1305aead(self): + "FSChaCha20Poly1305 AEAD test vectors." + for test_vector in FSAEAD_TESTS: + hex_plain, hex_aad, hex_key, msg_idx, hex_cipher = test_vector + plain = bytes.fromhex(hex_plain) + aad = bytes.fromhex(hex_aad) + key = bytes.fromhex(hex_key) + + enc_aead = FSChaCha20Poly1305(key) + dec_aead = FSChaCha20Poly1305(key) + + for _ in range(msg_idx): + enc_aead.encrypt(b"", b"") + ciphertext = enc_aead.encrypt(aad, plain) + self.assertEqual(hex_cipher, ciphertext.hex()) + + for _ in range(msg_idx): + dec_aead.decrypt(b"", bytes(16)) + plaintext = dec_aead.decrypt(aad, ciphertext) + self.assertEqual(plain, plaintext) diff --git a/test/functional/test_framework/crypto/chacha20.py b/test/functional/test_framework/crypto/chacha20.py new file mode 100644 index 0000000000..19b6698dfb --- /dev/null +++ b/test/functional/test_framework/crypto/chacha20.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of ChaCha20 cipher and FSChaCha20 for BIP 324 + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + +CHACHA20_INDICES = ( + (0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15), + (0, 5, 10, 15), (1, 6, 11, 12), (2, 7, 8, 13), (3, 4, 9, 14) +) + +CHACHA20_CONSTANTS = (0x61707865, 0x3320646e, 0x79622d32, 0x6b206574) +REKEY_INTERVAL = 224 # packets + + +def rotl32(v, bits): + """Rotate the 32-bit value v left by bits bits.""" + bits %= 32 # Make sure the term below does not throw an exception + return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) + + +def chacha20_doubleround(s): + """Apply a ChaCha20 double round to 16-element state array s. + See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 + """ + for a, b, c, d in CHACHA20_INDICES: + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 16) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 12) + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rotl32(s[d] ^ s[a], 8) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rotl32(s[b] ^ s[c], 7) + + +def chacha20_block(key, nonce, cnt): + """Compute the 64-byte output of the ChaCha20 block function. + Takes as input a 32-byte key, 12-byte nonce, and 32-bit integer counter. + """ + # Initial state. + init = [0] * 16 + init[:4] = CHACHA20_CONSTANTS[:4] + init[4:12] = [int.from_bytes(key[i:i+4], 'little') for i in range(0, 32, 4)] + init[12] = cnt + init[13:16] = [int.from_bytes(nonce[i:i+4], 'little') for i in range(0, 12, 4)] + # Perform 20 rounds. + state = list(init) + for _ in range(10): + chacha20_doubleround(state) + # Add initial values back into state. + for i in range(16): + state[i] = (state[i] + init[i]) & 0xffffffff + # Produce byte output + return b''.join(state[i].to_bytes(4, 'little') for i in range(16)) + +class FSChaCha20: + """Rekeying wrapper stream cipher around ChaCha20.""" + def __init__(self, initial_key, rekey_interval=REKEY_INTERVAL): + self._key = initial_key + self._rekey_interval = rekey_interval + self._block_counter = 0 + self._chunk_counter = 0 + self._keystream = b'' + + def _get_keystream_bytes(self, nbytes): + while len(self._keystream) < nbytes: + nonce = ((0).to_bytes(4, 'little') + (self._chunk_counter // self._rekey_interval).to_bytes(8, 'little')) + self._keystream += chacha20_block(self._key, nonce, self._block_counter) + self._block_counter += 1 + ret = self._keystream[:nbytes] + self._keystream = self._keystream[nbytes:] + return ret + + def crypt(self, chunk): + ks = self._get_keystream_bytes(len(chunk)) + ret = bytes([ks[i] ^ chunk[i] for i in range(len(chunk))]) + if ((self._chunk_counter + 1) % self._rekey_interval) == 0: + self._key = self._get_keystream_bytes(32) + self._block_counter = 0 + self._keystream = b'' + self._chunk_counter += 1 + return ret + + +# Test vectors from RFC7539/8439 consisting of 32 byte key, 12 byte nonce, block counter +# and 64 byte output after applying `chacha20_block` function +CHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0x09000000, 0x4a000000], 1, + "10f1e7e4d13b5915500fdd1fa32071c4c7d1f4c733c068030422aa9ac3d46c4e" + "d2826446079faa0914c2d705d98b02a2b5129cd1de164eb9cbd083e8a2503c4e"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7" + "da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0], 1, + "9f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32ee7aed" + "29b721769ce64e43d57133b074d839d531ed1f28510afb45ace10a1f4b794d6f"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 1, + "3aeb5224ecf849929b9d828db1ced4dd832025e8018b8160b82284f3c949aa5a" + "8eca00bbb4a73bdad192b5c42f73f2fd4e273644c8b36125a64addeb006c13a0"], + ["00ff000000000000000000000000000000000000000000000000000000000000", [0, 0], 2, + "72d54dfbf12ec44b362692df94137f328fea8da73990265ec1bbbea1ae9af0ca" + "13b25aa26cb4a648cb9b9d1be65b2c0924a66c54d545ec1b7374f4872e99f096"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 0x200000000000000], 0, + "c2c64d378cd536374ae204b9ef933fcd1a8b2288b3dfa49672ab765b54ee27c7" + "8a970e0e955c14f3a88e741b97c286f75f8fc299e8148362fa198a39531bed6d"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x4a000000], 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf78" + "8a3b0aa372600a92b57974cded2b9334794cba40c63e34cdea212c4cf07d41b7"], + ["0000000000000000000000000000000000000000000000000000000000000001", [0, 0], 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41" + "bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963"], + ["0000000000000000000000000000000000000000000000000000000000000000", [0, 1], 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32" + "111e4caf237ee53ca8ad6426194a88545ddc497a0b466e7d6bbdb0041b2f586b"], + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", [0, 0x0706050403020100], 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c1" + "34a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a"], +] + +FSCHACHA20_TESTS = [ + ["000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "0000000000000000000000000000000000000000000000000000000000000000", 256, + "a93df4ef03011f3db95f60d996e1785df5de38fc39bfcb663a47bb5561928349"], + ["01", "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 5, "ea"], + ["e93fdb5c762804b9a706816aca31e35b11d2aa3080108ef46a5b1f1508819c0a", + "8ec4c3ccdaea336bdeb245636970be01266509b33f3d2642504eaf412206207a", 4096, + "8bfaa4eacff308fdb4a94a5ff25bd9d0c1f84b77f81239f67ff39d6e1ac280c9"], +] + + +class TestFrameworkChacha(unittest.TestCase): + def test_chacha20(self): + """ChaCha20 test vectors.""" + for test_vector in CHACHA20_TESTS: + hex_key, nonce, counter, hex_output = test_vector + key = bytes.fromhex(hex_key) + nonce_bytes = nonce[0].to_bytes(4, 'little') + nonce[1].to_bytes(8, 'little') + keystream = chacha20_block(key, nonce_bytes, counter) + self.assertEqual(hex_output, keystream.hex()) + + def test_fschacha20(self): + """FSChaCha20 test vectors.""" + for test_vector in FSCHACHA20_TESTS: + hex_plaintext, hex_key, rekey_interval, hex_ciphertext_after_rotation = test_vector + plaintext = bytes.fromhex(hex_plaintext) + key = bytes.fromhex(hex_key) + fsc20 = FSChaCha20(key, rekey_interval) + for _ in range(rekey_interval): + fsc20.crypt(plaintext) + + ciphertext = fsc20.crypt(plaintext) + self.assertEqual(hex_ciphertext_after_rotation, ciphertext.hex()) diff --git a/test/functional/test_framework/ellswift.py b/test/functional/test_framework/crypto/ellswift.py index 97b10118e6..429b7b9f4d 100644 --- a/test/functional/test_framework/ellswift.py +++ b/test/functional/test_framework/crypto/ellswift.py @@ -12,7 +12,7 @@ import os import random import unittest -from test_framework.secp256k1 import FE, G, GE +from test_framework.crypto.secp256k1 import FE, G, GE # Precomputed constant square root of -3 (mod p). MINUS_3_SQRT = FE(-3).sqrt() diff --git a/test/functional/test_framework/ellswift_decode_test_vectors.csv b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv index 1bab96b721..1bab96b721 100644 --- a/test/functional/test_framework/ellswift_decode_test_vectors.csv +++ b/test/functional/test_framework/crypto/ellswift_decode_test_vectors.csv diff --git a/test/functional/test_framework/crypto/hkdf.py b/test/functional/test_framework/crypto/hkdf.py new file mode 100644 index 0000000000..7e8958733c --- /dev/null +++ b/test/functional/test_framework/crypto/hkdf.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only HKDF-SHA256 implementation + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import hashlib +import hmac + + +def hmac_sha256(key, data): + """Compute HMAC-SHA256 from specified byte arrays key and data.""" + return hmac.new(key, data, hashlib.sha256).digest() + + +def hkdf_sha256(length, ikm, salt, info): + """Derive a key using HKDF-SHA256.""" + if len(salt) == 0: + salt = bytes([0] * 32) + prk = hmac_sha256(salt, ikm) + t = b"" + okm = b"" + for i in range((length + 32 - 1) // 32): + t = hmac_sha256(prk, t + info + bytes([i + 1])) + okm += t + return okm[:length] diff --git a/test/functional/test_framework/crypto/muhash.py b/test/functional/test_framework/crypto/muhash.py new file mode 100644 index 0000000000..09241f6203 --- /dev/null +++ b/test/functional/test_framework/crypto/muhash.py @@ -0,0 +1,55 @@ +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Native Python MuHash3072 implementation.""" + +import hashlib +import unittest + +from .chacha20 import chacha20_block + +def data_to_num3072(data): + """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" + bytes384 = b"" + for counter in range(6): + bytes384 += chacha20_block(data, bytes(12), counter) + return int.from_bytes(bytes384, 'little') + +class MuHash3072: + """Class representing the MuHash3072 computation of a set. + + See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html + """ + + MODULUS = 2**3072 - 1103717 + + def __init__(self): + """Initialize for an empty set.""" + self.numerator = 1 + self.denominator = 1 + + def insert(self, data): + """Insert a byte array data in the set.""" + data_hash = hashlib.sha256(data).digest() + self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS + + def remove(self, data): + """Remove a byte array from the set.""" + data_hash = hashlib.sha256(data).digest() + self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS + + def digest(self): + """Extract the final hash. Does not modify this object.""" + val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS + bytes384 = val.to_bytes(384, 'little') + return hashlib.sha256(bytes384).digest() + +class TestFrameworkMuhash(unittest.TestCase): + def test_muhash(self): + muhash = MuHash3072() + muhash.insert(b'\x00' * 32) + muhash.insert((b'\x01' + b'\x00' * 31)) + muhash.remove((b'\x02' + b'\x00' * 31)) + finalized = muhash.digest() + # This mirrors the result in the C++ MuHash3072 unit test + self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") diff --git a/test/functional/test_framework/crypto/poly1305.py b/test/functional/test_framework/crypto/poly1305.py new file mode 100644 index 0000000000..967b90254d --- /dev/null +++ b/test/functional/test_framework/crypto/poly1305.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Test-only implementation of Poly1305 authenticator + +It is designed for ease of understanding, not performance. + +WARNING: This code is slow and trivially vulnerable to side channel attacks. Do not use for +anything but tests. +""" + +import unittest + + +class Poly1305: + """Class representing a running poly1305 computation.""" + MODULUS = 2**130 - 5 + + def __init__(self, key): + self.r = int.from_bytes(key[:16], 'little') & 0xffffffc0ffffffc0ffffffc0fffffff + self.s = int.from_bytes(key[16:], 'little') + + def tag(self, data): + """Compute the poly1305 tag.""" + acc, length = 0, len(data) + for i in range((length + 15) // 16): + chunk = data[i * 16:min(length, (i + 1) * 16)] + val = int.from_bytes(chunk, 'little') + 256**len(chunk) + acc = (self.r * (acc + val)) % Poly1305.MODULUS + return ((acc + self.s) & 0xffffffffffffffffffffffffffffffff).to_bytes(16, 'little') + + +# Test vectors from RFC7539/8439 consisting of message to be authenticated, 32 byte key and computed 16 byte tag +POLY1305_TESTS = [ + # RFC 7539, section 2.5.2. + ["43727970746f6772617068696320466f72756d2052657365617263682047726f7570", + "85d6be7857556d337f4452fe42d506a80103808afb0db2fd4abff6af4149f51b", + "a8061dc1305136c6c22b8baf0c0127a9"], + # RFC 7539, section A.3. + ["00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "0000000000000000000000000000000036e5f6b5c5e06070f0efca96227a863e", + "36e5f6b5c5e06070f0efca96227a863e"], + ["416e79207375626d697373696f6e20746f20746865204945544620696e74656e6465642062792074686520436f6e747269627" + "5746f7220666f72207075626c69636174696f6e20617320616c6c206f722070617274206f6620616e204945544620496e7465" + "726e65742d4472616674206f722052464320616e6420616e792073746174656d656e74206d6164652077697468696e2074686" + "520636f6e74657874206f6620616e204945544620616374697669747920697320636f6e7369646572656420616e2022494554" + "4620436f6e747269627574696f6e222e20537563682073746174656d656e747320696e636c756465206f72616c20737461746" + "56d656e747320696e20494554462073657373696f6e732c2061732077656c6c206173207772697474656e20616e6420656c65" + "6374726f6e696320636f6d6d756e69636174696f6e73206d61646520617420616e792074696d65206f7220706c6163652c207" + "768696368206172652061646472657373656420746f", + "36e5f6b5c5e06070f0efca96227a863e00000000000000000000000000000000", + "f3477e7cd95417af89a6b8794c310cf0"], + ["2754776173206272696c6c69672c20616e642074686520736c6974687920746f7665730a446964206779726520616e6420676" + "96d626c6520696e2074686520776162653a0a416c6c206d696d737920776572652074686520626f726f676f7665732c0a416e" + "6420746865206d6f6d65207261746873206f757467726162652e", + "1c9240a5eb55d38af333888604f6b5f0473917c1402b80099dca5cbc207075c0", + "4541669a7eaaee61e708dc7cbcc5eb62"], + ["ffffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "03000000000000000000000000000000"], + ["02000000000000000000000000000000", + "02000000000000000000000000000000ffffffffffffffffffffffffffffffff", + "03000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffff0ffffffffffffffffffffffffffffff11000000000000000000000000000000", + "0100000000000000000000000000000000000000000000000000000000000000", + "05000000000000000000000000000000"], + ["fffffffffffffffffffffffffffffffffbfefefefefefefefefefefefefefefe01010101010101010101010101010101", + "0100000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000"], + ["fdffffffffffffffffffffffffffffff", + "0200000000000000000000000000000000000000000000000000000000000000", + "faffffffffffffffffffffffffffffff"], + ["e33594d7505e43b900000000000000003394d7505e4379cd01000000000000000000000000000000000000000000000001000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "14000000000000005500000000000000"], + ["e33594d7505e43b900000000000000003394d7505e4379cd010000000000000000000000000000000000000000000000", + "0100000000000000040000000000000000000000000000000000000000000000", + "13000000000000000000000000000000"], +] + + +class TestFrameworkPoly1305(unittest.TestCase): + def test_poly1305(self): + """Poly1305 test vectors.""" + for test_vector in POLY1305_TESTS: + hex_message, hex_key, hex_tag = test_vector + message = bytes.fromhex(hex_message) + key = bytes.fromhex(hex_key) + tag = bytes.fromhex(hex_tag) + comp_tag = Poly1305(key).tag(message) + self.assertEqual(tag, comp_tag) diff --git a/test/functional/test_framework/ripemd160.py b/test/functional/test_framework/crypto/ripemd160.py index 12801364b4..12801364b4 100644 --- a/test/functional/test_framework/ripemd160.py +++ b/test/functional/test_framework/crypto/ripemd160.py diff --git a/test/functional/test_framework/secp256k1.py b/test/functional/test_framework/crypto/secp256k1.py index 2e9e419da5..2e9e419da5 100644 --- a/test/functional/test_framework/secp256k1.py +++ b/test/functional/test_framework/crypto/secp256k1.py diff --git a/test/functional/test_framework/siphash.py b/test/functional/test_framework/crypto/siphash.py index bd13b2c948..bd13b2c948 100644 --- a/test/functional/test_framework/siphash.py +++ b/test/functional/test_framework/crypto/siphash.py diff --git a/test/functional/test_framework/xswiftec_inv_test_vectors.csv b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv index 138c4cf85c..138c4cf85c 100644 --- a/test/functional/test_framework/xswiftec_inv_test_vectors.csv +++ b/test/functional/test_framework/crypto/xswiftec_inv_test_vectors.csv diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 6c1892539f..06252f8996 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -13,7 +13,7 @@ import os import random import unittest -from test_framework import secp256k1 +from test_framework.crypto import secp256k1 # Point with no known discrete log. H_POINT = "50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0" diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8f3aea8785..d008cb39aa 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -29,7 +29,7 @@ import struct import time import unittest -from test_framework.siphash import siphash256 +from test_framework.crypto.siphash import siphash256 from test_framework.util import assert_equal MAX_LOCATOR_SZ = 101 diff --git a/test/functional/test_framework/muhash.py b/test/functional/test_framework/muhash.py deleted file mode 100644 index 0d96114e3e..0000000000 --- a/test/functional/test_framework/muhash.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2020 Pieter Wuille -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Native Python MuHash3072 implementation.""" - -import hashlib -import unittest - -def rot32(v, bits): - """Rotate the 32-bit value v left by bits bits.""" - bits %= 32 # Make sure the term below does not throw an exception - return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) - -def chacha20_doubleround(s): - """Apply a ChaCha20 double round to 16-element state array s. - - See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 - """ - QUARTER_ROUNDS = [(0, 4, 8, 12), - (1, 5, 9, 13), - (2, 6, 10, 14), - (3, 7, 11, 15), - (0, 5, 10, 15), - (1, 6, 11, 12), - (2, 7, 8, 13), - (3, 4, 9, 14)] - - for a, b, c, d in QUARTER_ROUNDS: - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 16) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 12) - s[a] = (s[a] + s[b]) & 0xffffffff - s[d] = rot32(s[d] ^ s[a], 8) - s[c] = (s[c] + s[d]) & 0xffffffff - s[b] = rot32(s[b] ^ s[c], 7) - -def chacha20_32_to_384(key32): - """Specialized ChaCha20 implementation with 32-byte key, 0 IV, 384-byte output.""" - # See RFC 8439 section 2.3 for chacha20 parameters - CONSTANTS = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574] - - key_bytes = [0]*8 - for i in range(8): - key_bytes[i] = int.from_bytes(key32[(4 * i):(4 * (i+1))], 'little') - - INITIALIZATION_VECTOR = [0] * 4 - init = CONSTANTS + key_bytes + INITIALIZATION_VECTOR - out = bytearray() - for counter in range(6): - init[12] = counter - s = init.copy() - for _ in range(10): - chacha20_doubleround(s) - for i in range(16): - out.extend(((s[i] + init[i]) & 0xffffffff).to_bytes(4, 'little')) - return bytes(out) - -def data_to_num3072(data): - """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" - bytes384 = chacha20_32_to_384(data) - return int.from_bytes(bytes384, 'little') - -class MuHash3072: - """Class representing the MuHash3072 computation of a set. - - See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html - """ - - MODULUS = 2**3072 - 1103717 - - def __init__(self): - """Initialize for an empty set.""" - self.numerator = 1 - self.denominator = 1 - - def insert(self, data): - """Insert a byte array data in the set.""" - data_hash = hashlib.sha256(data).digest() - self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS - - def remove(self, data): - """Remove a byte array from the set.""" - data_hash = hashlib.sha256(data).digest() - self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS - - def digest(self): - """Extract the final hash. Does not modify this object.""" - val = (self.numerator * pow(self.denominator, -1, self.MODULUS)) % self.MODULUS - bytes384 = val.to_bytes(384, 'little') - return hashlib.sha256(bytes384).digest() - -class TestFrameworkMuhash(unittest.TestCase): - def test_muhash(self): - muhash = MuHash3072() - muhash.insert(b'\x00' * 32) - muhash.insert((b'\x01' + b'\x00' * 31)) - muhash.remove((b'\x02' + b'\x00' * 31)) - finalized = muhash.digest() - # This mirrors the result in the C++ MuHash3072 unit test - self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") - - def test_chacha20(self): - def chacha_check(key, result): - self.assertEqual(chacha20_32_to_384(key)[:64].hex(), result) - - # Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 - # Since the nonce is hardcoded to 0 in our function we only use those vectors. - chacha_check([0]*32, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586") - chacha_check([0]*31 + [1], "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963") diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 352becb367..b1ed97b794 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -364,8 +364,8 @@ class P2PInterface(P2PConnection): vt.addrFrom.port = 0 self.on_connection_send_msg = vt # Will be sent in connection_made callback - def peer_connect(self, *args, services=P2P_SERVICES, send_version=True, **kwargs): - create_conn = super().peer_connect(*args, **kwargs) + def peer_connect(self, *, services=P2P_SERVICES, send_version, **kwargs): + create_conn = super().peer_connect(**kwargs) if send_version: self.peer_connect_send_version(services) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 17a954cb22..f4628bf4af 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -24,7 +24,7 @@ from .messages import ( uint256_from_str, ) -from .ripemd160 import ripemd160 +from .crypto.ripemd160 import ripemd160 MAX_SCRIPT_ELEMENT_SIZE = 520 MAX_PUBKEYS_PER_MULTI_A = 999 diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index b6af71d85c..281469f9ed 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -634,7 +634,7 @@ class TestNode(): assert_msg += "with expected error " + expected_msg self._raise_assertion_error(assert_msg) - def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): + def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, send_version=True, **kwargs): """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also @@ -645,9 +645,11 @@ class TestNode(): kwargs['dstaddr'] = '127.0.0.1' p2p_conn.p2p_connected_to_node = True - p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() + p2p_conn.peer_connect(**kwargs, send_version=send_version, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False) + if send_version: + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: # Wait for the node to send us the version and verack p2p_conn.wait_for_verack() @@ -701,6 +703,7 @@ class TestNode(): p2p_conn.wait_for_connect() self.p2ps.append(p2p_conn) + p2p_conn.wait_until(lambda: not p2p_conn.on_connection_send_msg) if wait_for_verack: p2p_conn.wait_for_verack() p2p_conn.sync_with_ping() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2b6be93bdf..35c662b0d9 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -73,12 +73,15 @@ TEST_EXIT_SKIPPED = 77 # the output of `git grep unittest.TestCase ./test/functional/test_framework` TEST_FRAMEWORK_MODULES = [ "address", + "crypto.bip324_cipher", "blocktools", - "ellswift", + "crypto.chacha20", + "crypto.ellswift", "key", "messages", - "muhash", - "ripemd160", + "crypto.muhash", + "crypto.poly1305", + "crypto.ripemd160", "script", "segwit_addr", ] diff --git a/test/functional/wallet_miniscript.py b/test/functional/wallet_miniscript.py index e6c8ad545c..67e1283902 100755 --- a/test/functional/wallet_miniscript.py +++ b/test/functional/wallet_miniscript.py @@ -208,6 +208,7 @@ class WalletMiniscriptTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.rpc_timeout = 180 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index ec74f7705c..e72977fac0 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -16,11 +16,14 @@ import sys def get_fuzz_env(*, target, source_dir): + symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") return { 'FUZZ': target, 'UBSAN_OPTIONS': f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'UBSAN_SYMBOLIZER_PATH':symbolizer, "ASAN_OPTIONS": "detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1", + 'ASAN_SYMBOLIZER_PATH':symbolizer, } diff --git a/test/lint/README.md b/test/lint/README.md index c0889b59af..484008298b 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -13,6 +13,14 @@ DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./ Building the container can be done every time, because it is fast when the result is cached and it prevents issues when the image changes. +test runner +=========== + +To run the checks in the test runner outside the docker, use: + +```sh +( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run ) +``` check-doc.py ============ diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index d22dd9d996..f55d0f8cb7 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb', '-zapwallettxes']) +SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def lint_missing_argument_documentation(): diff --git a/test/lint/test_runner/Cargo.lock b/test/lint/test_runner/Cargo.lock new file mode 100644 index 0000000000..ca83aa9331 --- /dev/null +++ b/test/lint/test_runner/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "test_runner" +version = "0.1.0" diff --git a/test/lint/test_runner/Cargo.toml b/test/lint/test_runner/Cargo.toml new file mode 100644 index 0000000000..053ce43d6c --- /dev/null +++ b/test/lint/test_runner/Cargo.toml @@ -0,0 +1,12 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +[package] +name = "test_runner" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs new file mode 100644 index 0000000000..b7ec9ee3b2 --- /dev/null +++ b/test/lint/test_runner/src/main.rs @@ -0,0 +1,77 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +use std::env; +use std::path::PathBuf; +use std::process::Command; +use std::process::ExitCode; + +use String as LintError; + +/// Return the git command +fn git() -> Command { + Command::new("git") +} + +/// Return stdout +fn check_output(cmd: &mut std::process::Command) -> Result<String, LintError> { + let out = cmd.output().expect("command error"); + if !out.status.success() { + return Err(String::from_utf8_lossy(&out.stderr).to_string()); + } + Ok(String::from_utf8(out.stdout) + .map_err(|e| format!("{e}"))? + .trim() + .to_string()) +} + +/// Return the git root as utf8, or panic +fn get_git_root() -> String { + check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap() +} + +fn lint_std_filesystem() -> Result<(), LintError> { + let found = git() + .args([ + "grep", + "std::filesystem", + "--", + "./src/", + ":(exclude)src/util/fs.h", + ]) + .status() + .expect("command error") + .success(); + if found { + Err(r#" +^^^ +Direct use of std::filesystem may be dangerous and buggy. Please include <util/fs.h> and use the +fs:: namespace, which has unsafe filesystem functions marked as deleted. + "# + .to_string()) + } else { + Ok(()) + } +} + +fn main() -> ExitCode { + let test_list = [("std::filesystem check", lint_std_filesystem)]; + + let git_root = PathBuf::from(get_git_root()); + + let mut test_failed = false; + for (lint_name, lint_fn) in test_list { + // chdir to root before each lint test + env::set_current_dir(&git_root).unwrap(); + if let Err(err) = lint_fn() { + println!("{err}\n^---- Failure generated from {lint_name}!"); + test_failed = true; + } + } + if test_failed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 533e2eae51..63d2215aab 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -39,28 +39,36 @@ shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. unsigned-integer-overflow:arith_uint256.h -unsigned-integer-overflow:common/bloom.cpp -unsigned-integer-overflow:coins.cpp -unsigned-integer-overflow:compressor.cpp +unsigned-integer-overflow:CBloomFilter::Hash +unsigned-integer-overflow:CRollingBloomFilter::insert +unsigned-integer-overflow:RollingBloomHash +unsigned-integer-overflow:CCoinsViewCache::AddCoin +unsigned-integer-overflow:CCoinsViewCache::BatchWrite +unsigned-integer-overflow:CCoinsViewCache::DynamicMemoryUsage +unsigned-integer-overflow:CCoinsViewCache::SpendCoin +unsigned-integer-overflow:CCoinsViewCache::Uncache +unsigned-integer-overflow:CompressAmount +unsigned-integer-overflow:DecompressAmount unsigned-integer-overflow:crypto/ -unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:policy/fees.cpp +unsigned-integer-overflow:MurmurHash3 +unsigned-integer-overflow:CBlockPolicyEstimator::processBlockTx +unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:xoroshiro128plusplus.h -implicit-integer-sign-change:compat/stdin.cpp +implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h implicit-integer-sign-change:crypto/ -implicit-integer-sign-change:policy/fees.cpp +implicit-integer-sign-change:TxConfirmStats::removeTx implicit-integer-sign-change:prevector.h -implicit-integer-sign-change:script/bitcoinconsensus.cpp -implicit-integer-sign-change:script/interpreter.cpp +implicit-integer-sign-change:verify_flags +implicit-integer-sign-change:EvalScript implicit-integer-sign-change:serialize.h implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp shift-base:crypto/ -shift-base:hash.cpp +shift-base:ROTL32 shift-base:streams.h -shift-base:util/bip32.cpp +shift-base:FormatHDKeypath shift-base:xoroshiro128plusplus.h |