diff options
81 files changed, 1617 insertions, 568 deletions
diff --git a/Makefile.am b/Makefile.am index d84dab1842..3ac698b96f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -58,6 +58,7 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ + $(top_srcdir)/contrib/devtools/utils.py \ $(top_srcdir)/contrib/devtools/pixie.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ @@ -366,14 +367,14 @@ clean-local: clean-docs test-security-check: if TARGET_DARWIN - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 1418dfbc51..8869b2a083 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -13,5 +13,4 @@ export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" -export RUN_SECURITY_TESTS="true" export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 9c57eba62a..b3e967c898 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -14,6 +14,6 @@ export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude fe export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/configure.ac b/configure.ac index 94f01a51a6..6f78e8dec4 100644 --- a/configure.ac +++ b/configure.ac @@ -900,6 +900,7 @@ if test x$use_hardening != xno; then ]) fi + AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 56e4313d78..61f727fa63 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -3,21 +3,22 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -A script to check that the executables produced by gitian only contain -certain symbols and are only linked against allowed libraries. +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. Example usage: - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py + find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import sys -import os from typing import List, Optional import lief import pixie +from utils import determine_wellknown_cmd + # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # # - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) @@ -60,7 +61,6 @@ IGNORE_EXPORTS = { '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } -CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { @@ -140,7 +140,7 @@ class CPPFilt(object): Use a pipe to the 'c++filt' command. ''' def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index c079fe5b4d..14058e2cc8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -9,6 +9,8 @@ import os import subprocess import unittest +from utils import determine_wellknown_cmd + def write_testcode(filename): with open(filename, 'w', encoding="utf8") as f: f.write(''' @@ -25,7 +27,7 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - subprocess.run([cc,source,'-o',executable] + options, check=True) + subprocess.run([*cc,source,'-o',executable] + options, check=True) p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) @@ -33,7 +35,7 @@ class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), @@ -54,18 +56,20 @@ class TestSecurityChecks(unittest.TestCase): def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'x86_64-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), (0, '')) clean_files(source, executable) @@ -73,7 +77,7 @@ class TestSecurityChecks(unittest.TestCase): def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), @@ -95,4 +99,3 @@ class TestSecurityChecks(unittest.TestCase): if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 6ce2fa3560..7d83c5f751 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -7,41 +7,51 @@ Test script for symbol-check.py ''' import os import subprocess +from typing import List import unittest -def call_symbol_check(cc, source, executable, options): - subprocess.run([cc,source,'-o',executable] + options, check=True) +from utils import determine_wellknown_cmd + +def call_symbol_check(cc: List[str], source, executable, options): + subprocess.run([*cc,source,'-o',executable] + options, check=True) p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) +def get_machine(cc: List[str]): + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + return p.stdout.rstrip() + class TestSymbolChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') + + # there's no way to do this test for RISC-V at the moment; we build for + # RISC-V in a glibc 2.27 envinonment and we allow all symbols from 2.27. + if 'riscv' in get_machine(cc): + self.skipTest("test not available for RISC-V") - # renameat2 was introduced in GLIBC 2.28, so is newer than the upper limit - # of glibc for all platforms + # nextup was introduced in GLIBC 2.24, so is newer than our supported + # glibc (2.17), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE - #include <stdio.h> - #include <linux/fs.h> + #include <math.h> - int renameat2(int olddirfd, const char *oldpath, - int newdirfd, const char *newpath, unsigned int flags); + double nextup(double x); int main() { - renameat2(0, "test", 0, "test_", RENAME_EXCHANGE); + nextup(3.14); return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, []), - (1, executable + ': symbol renameat2 from unsupported version GLIBC_2.28\n' + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -82,7 +92,7 @@ class TestSymbolChecks(unittest.TestCase): def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -96,7 +106,7 @@ class TestSymbolChecks(unittest.TestCase): ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) @@ -113,7 +123,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, f'{executable}: failed MIN_OS SDK')) source = 'test3.c' @@ -126,13 +136,13 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-mmacosx-version-min=10.14']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.14', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'x86_64-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -182,4 +192,3 @@ class TestSymbolChecks(unittest.TestCase): if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 0000000000..68ad1c3aba --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e457840d15..0b96949a6b 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -240,7 +240,7 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" case "$HOST" in - *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; + *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; esac # CFLAGS @@ -260,6 +260,13 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. +# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html +case "$HOST" in + *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; +esac + case "$HOST" in powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; esac @@ -292,10 +299,11 @@ mkdir -p "$DISTSRC" # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} - # Perform basic ELF security checks on a series of executables. + # Check that symbol/security checks tools are sane. + make test-security-check ${V:+V=1} + # Perform basic security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. + # Check that executables only contain allowed version symbols. make -C src --jobs=1 check-symbols ${V:+V=1} mkdir -p "$OUTDIR" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index f4df4855fc..e71cf52533 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -80,6 +80,10 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (("-rpath=") "-rpath-link=")) #t)))))))) +(define (make-binutils-with-mingw-w64-disable-flags xbinutils) + (package-with-extra-patches xbinutils + (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) + (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers @@ -168,7 +172,7 @@ desirable for building Bitcoin Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch new file mode 100644 index 0000000000..8f88eb9dfd --- /dev/null +++ b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch @@ -0,0 +1,171 @@ +Description: Add disable opposites to the security-related flags +Author: Stephen Kitt <skitt@debian.org> + +This patch adds "no-" variants to disable the various security flags: +"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". + +--- a/ld/emultempl/pe.em ++++ b/ld/emultempl/pe.em +@@ -259,9 +261,11 @@ + (OPTION_ENABLE_LONG_SECTION_NAMES + 1) + /* DLLCharacteristics flags. */ + #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) +-#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) + #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) +-#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) + #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) + #define OPTION_NO_BIND (OPTION_NO_SEH + 1) + #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) +@@ -271,6 +275,7 @@ + #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) + #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) + #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) ++#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) + + static void + gld${EMULATION_NAME}_add_options +@@ -342,8 +347,10 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -351,6 +358,7 @@ + {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -485,9 +494,12 @@ + in object files\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ + be called in this image\n")); +@@ -862,12 +874,21 @@ + case OPTION_ENABLE_RELOC_SECTION: + pe_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pe_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; +--- a/ld/emultempl/pep.em ++++ b/ld/emultempl/pep.em +@@ -237,9 +240,12 @@ + OPTION_ENABLE_LONG_SECTION_NAMES, + OPTION_DISABLE_LONG_SECTION_NAMES, + OPTION_HIGH_ENTROPY_VA, ++ OPTION_NO_HIGH_ENTROPY_VA, + OPTION_DYNAMIC_BASE, ++ OPTION_NO_DYNAMIC_BASE, + OPTION_FORCE_INTEGRITY, + OPTION_NX_COMPAT, ++ OPTION_NO_NX_COMPAT, + OPTION_NO_ISOLATION, + OPTION_NO_SEH, + OPTION_NO_BIND, +@@ -248,7 +254,8 @@ + OPTION_NO_INSERT_TIMESTAMP, + OPTION_TERMINAL_SERVER_AWARE, + OPTION_BUILD_ID, +- OPTION_ENABLE_RELOC_SECTION ++ OPTION_ENABLE_RELOC_SECTION, ++ OPTION_DISABLE_RELOC_SECTION + }; + + static void +@@ -315,9 +322,12 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, ++ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -327,6 +337,7 @@ + {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -448,11 +461,15 @@ + in object files\n")); + fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ + layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ + be called in this image\n")); +@@ -809,12 +826,24 @@ + case OPTION_ENABLE_RELOC_SECTION: + pep_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pep_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ /* fall through */ ++ case OPTION_NO_HIGH_ENTROPY_VA: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch index 1dfe3ba132..f327c464f3 100644 --- a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch +++ b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch @@ -1,3 +1,11 @@ +guix: repro: Sort find output in libtool for gcc-8 + +Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not +be reproducible and end up making the Bitcoin binaries non-reproducible +as well. + +See: https://reproducible-builds.org/docs/archives/#gnu-libtool + diff --git a/gcc/configure b/gcc/configure index 97ba7d7d69c..e37a96f0c0c 100755 --- a/gcc/configure diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch index 54a7824345..5c4d0c6ebe 100644 --- a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch +++ b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch @@ -1,3 +1,5 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 + commit 6b02af31e9a721bb15a11380cd22d53b621711f8 Author: Szabolcs Nagy <szabolcs.nagy@arm.com> Date: Wed Oct 18 17:26:23 2017 +0100 diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch index 5d7a148c62..11fe7fdc99 100644 --- a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch +++ b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch @@ -1,3 +1,5 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc + commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 Author: Florian Weimer <fweimer@redhat.com> Date: Wed Sep 21 10:45:32 2016 +0200 diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch index 39c995ffb5..d6217157ee 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch @@ -1,3 +1,5 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e + From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" <hjl.tools@gmail.com> Date: Mon, 31 Dec 2018 09:26:42 -0800 diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch index 5edf1b7c8e..f58406a7a0 100644 --- a/contrib/guix/patches/nsis-SConstruct-sde-support.patch +++ b/contrib/guix/patches/nsis-SConstruct-sde-support.patch @@ -1,3 +1,6 @@ +https://github.com/kichik/nsis/pull/13 +https://sourceforge.net/p/nsis/code/7248/ + diff --git a/SConstruct b/SConstruct index e8252c9..41786f2 100755 --- a/SConstruct diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index a62150a930..f7bfb6eb0a 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -678,11 +678,11 @@ vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 # manually added 2021-05 for minimal i2p bootstrap support -72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:8333 -c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:8333 -gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:8333 -h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:8333 -hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:8333 -pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:8333 -wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:8333 -zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:8333 +72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 diff --git a/depends/README.md b/depends/README.md index 50e1a32c70..4f3b6df487 100644 --- a/depends/README.md +++ b/depends/README.md @@ -87,6 +87,14 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu +### Install the required dependencies: M1-based macOS + +To be able to build the `qt` package, ensure that Rosetta 2 is installed: + +``` +softwareupdate --install-rosetta +``` + ### Dependency Options The following can be set when running make: `make FOO=bar` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 613aea438f..89fd506f13 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ OpenBSD build guide ====================== -(updated for OpenBSD 6.7) +(updated for OpenBSD 6.9) This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. @@ -67,9 +67,16 @@ export AUTOMAKE_VERSION=1.16 ``` Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. +Note that building with external signer support currently fails on OpenBSD, +hence you have to explicitely disable it by passing the parameter +`--disable-external-signer` to the configure script. +(Background: the feature requires the header-only library boost::process, which +is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls +and preprocessor defines like `waitid()` and `WEXITED` that are not available.) + To configure with wallet: ```bash -./configure --with-gui=no CC=cc CXX=c++ \ +./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake @@ -77,12 +84,12 @@ To configure with wallet: To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake ``` To configure with GUI: ```bash -./configure --with-gui=yes CC=cc CXX=c++ \ +./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake diff --git a/doc/i2p.md b/doc/i2p.md index 8b4607208a..27ef4d9d9f 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -70,3 +70,18 @@ RPC. Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) protocol to connect to the I2P network. Any I2P router that supports it can be used. + +## Ports in I2P and Bitcoin Core + +Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) +protocol. One particularity of SAM v3.1 is that it does not support ports, +unlike newer versions of SAM (v3.2 and up) that do support them and default the +port numbers to 0. From the point of view of peers that use newer versions of +SAM or other protocols that support ports, a SAM v3.1 peer is connecting to them +on port 0, from source port 0. + +To allow future upgrades to newer versions of SAM, Bitcoin Core sets its +listening port to 0 when listening for incoming I2P connections and advertises +its own I2P address with port 0. Furthermore, it will not attempt to connect to +I2P addresses with a non-zero port number because with SAM v3.1 the destination +port (`TO_PORT`) is always set to 0 and is not in the control of Bitcoin Core. diff --git a/doc/release-process.md b/doc/release-process.md index 3ead1181b9..75a574ee31 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -37,6 +37,12 @@ Release Process - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect that causes rejection of blocks in the past history. - Clear the release notes and move them to the wiki (see "Write the release notes" below). +- Translations on Transifex + - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-<RRR>x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. + - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. + - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. + - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/bitcoin/announcements/) as a template. + - Change the auto-update URL for the resource to `master`, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/master/src/qt/locale/bitcoin_en.xlf`. (Do this only after the previous steps, to prevent an auto-update from interfering.) #### After branch-off (on master) @@ -46,6 +52,8 @@ Release Process - Update the versions. - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. +- Translations on Transifex + - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release @@ -278,7 +286,7 @@ The `*-debug*` files generated by the gitian build contain debug symbols for troubleshooting by developers. It is assumed that anyone that is interested in debugging can run gitian to generate the files for themselves. To avoid end-user confusion about which file to pick, as well as save storage -space *do not upload these to the bitcoin.org server, nor put them in the torrent*. +space *do not upload these to the bitcoincore.org server, nor put them in the torrent*. - GPG-sign it, delete the unsigned file: ``` @@ -288,7 +296,7 @@ rm SHA256SUMS (the digest algorithm is forced to sha256 to avoid confusion of the `Hash:` header that GPG adds with the SHA256 used for the files) Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spurious/nonsensical entry. -- Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoin.org server +- Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoincore.org server into `/var/www/bin/bitcoin-core-${VERSION}` - A `.torrent` will appear in the directory after a few minutes. Optionally help seed this torrent. To get the `magnet:` URI use: @@ -296,24 +304,9 @@ Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spur transmission-show -m <torrent file> ``` Insert the magnet URI into the announcement sent to mailing lists. This permits -people without access to `bitcoin.org` to download the binary distribution. +people without access to `bitcoincore.org` to download the binary distribution. Also put it into the `optional_magnetlink:` slot in the YAML file for -bitcoin.org (see below for bitcoin.org update instructions). - -- Update bitcoin.org version - - - First, check to see if the Bitcoin.org maintainers have prepared a - release: https://github.com/bitcoin-dot-org/bitcoin.org/labels/Core - - - If they have, it will have previously failed their CI - checks because the final release files weren't uploaded. - Trigger a CI rebuild---if it passes, merge. - - - If they have not prepared a release, follow the Bitcoin.org release - instructions: https://github.com/bitcoin-dot-org/bitcoin.org/blob/master/docs/adding-events-release-notes-and-alerts.md#release-notes - - - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well - as update the OS download links. +bitcoincore.org. - Update other repositories and websites for new version diff --git a/doc/translation_process.md b/doc/translation_process.md index f132693264..97a8fbfff2 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -63,17 +63,12 @@ username = USERNAME The Transifex Bitcoin project config file is included as part of the repo. It can be found at `.tx/config`, however you shouldn’t need to change anything. ### Synchronising translations -To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). -1. `python3 ../bitcoin-maintainer-tools/update-translations.py` -2. `git add` new translations from `src/qt/locale/` -3. Update `src/qt/bitcoin_locale.qrc` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ <file alias="\2">locale\/\1.qm<\/file>/' +To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). To use it and commit the result, simply do: + ``` -4. Update `src/Makefile.qt_locale.include` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ qt\/locale\/\1.ts \\/' +python3 ../bitcoin-maintainer-tools/update-translations.py +git commit -a ``` **Do not directly download translations** one by one from the Transifex website, as we do a few post-processing steps before committing the translations. diff --git a/src/Makefile.am b/src/Makefile.am index 37ba5ad75b..7de5fb36ed 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -814,23 +814,23 @@ clean-local: check-symbols: $(bin_PROGRAMS) if TARGET_DARWIN @echo "Checking macOS dynamic libraries..." - $(AM_V_at) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif if TARGET_WINDOWS @echo "Checking Windows dynamic libraries..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif -if GLIBC_BACK_COMPAT +if TARGET_LINUX @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif libbitcoin_ipc_mpgen_input = \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 56b8ca8ce6..2a8e4a0aac 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -35,6 +35,7 @@ bench_bench_bitcoin_SOURCES = \ bench/mempool_stress.cpp \ bench/nanobench.h \ bench/nanobench.cpp \ + bench/peer_eviction.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 8f702b5a8c..389d106164 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -6,6 +6,7 @@ #include <addrman.h> #include <hash.h> +#include <i2p.h> #include <logging.h> #include <netaddress.h> #include <serialize.h> @@ -77,6 +78,38 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } +void CAddrMan::RemoveInvalid() +{ + for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + const auto id = vvNew[bucket][i]; + if (id != -1 && !mapInfo[id].IsValid()) { + ClearNew(bucket, i); + } + } + } + + for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; ++bucket) { + for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + const auto id = vvTried[bucket][i]; + if (id == -1) { + continue; + } + const auto& addr_info = mapInfo[id]; + if (addr_info.IsValid()) { + continue; + } + vvTried[bucket][i] = -1; + --nTried; + SwapRandom(addr_info.nRandomPos, vRandom.size() - 1); + vRandom.pop_back(); + mapAddr.erase(addr_info); + mapInfo.erase(id); + m_tried_collisions.erase(id); + } + } +} + CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) { AssertLockHeld(cs); @@ -699,3 +732,100 @@ std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) } return bits; } + +void CAddrMan::ResetI2PPorts() +{ + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + const auto id = vvNew[bucket][i]; + if (id == -1) { + continue; + } + auto it = mapInfo.find(id); + if (it == mapInfo.end()) { + return; + } + auto& addr_info = it->second; + if (!addr_info.IsI2P() || addr_info.GetPort() == I2P_SAM31_PORT) { + continue; + } + + auto addr_info_newport = addr_info; + // The below changes addr_info_newport.GetKey(), which is used in finding a + // bucket and a position within that bucket. So a re-bucketing may be necessary. + addr_info_newport.port = I2P_SAM31_PORT; + + // Reposition entries of vvNew within the same bucket because we don't know the source + // address which led to the decision to store the entry in vvNew[bucket] so we can't + // re-evaluate that decision, but even if we could, CAddrInfo::GetNewBucket() does not + // use CAddrInfo::GetKey() so it would end up in the same bucket as before the port + // change. + const auto i_target = addr_info_newport.GetBucketPosition(nKey, true, bucket); + + if (i_target == i) { // No need to re-position. + addr_info = addr_info_newport; + continue; + } + + // Reposition from i to i_target, removing the entry from i_target (if any). + ClearNew(bucket, i_target); + vvNew[bucket][i_target] = id; + vvNew[bucket][i] = -1; + addr_info = addr_info_newport; + } + } + + for (int bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; ++bucket) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + const auto id = vvTried[bucket][i]; + if (id == -1) { + continue; + } + auto it = mapInfo.find(id); + if (it == mapInfo.end()) { + return; + } + auto& addr_info = it->second; + if (!addr_info.IsI2P() || addr_info.GetPort() == I2P_SAM31_PORT) { + continue; + } + + auto addr_info_newport = addr_info; + // The below changes addr_info_newport.GetKey(), which is used in finding a + // bucket and a position within that bucket. So a re-bucketing may be necessary. + addr_info_newport.port = I2P_SAM31_PORT; + + const auto bucket_target = addr_info_newport.GetTriedBucket(nKey, m_asmap); + const auto i_target = addr_info_newport.GetBucketPosition(nKey, false, bucket_target); + + if (bucket_target == bucket && i_target == i) { // No need to re-position. + addr_info = addr_info_newport; + continue; + } + + // Reposition from (bucket, i) to (bucket_target, i_target). If the latter is + // occupied, then move the entry from there to vvNew. + + const auto old_target_id = vvTried[bucket_target][i_target]; + if (old_target_id != -1) { + CAddrInfo& old_target_info = mapInfo[old_target_id]; + + old_target_info.fInTried = false; + vvTried[bucket_target][i_target] = -1; + --nTried; + + const auto new_bucket = old_target_info.GetNewBucket(nKey, m_asmap); + const auto new_bucket_i = old_target_info.GetBucketPosition(nKey, true, new_bucket); + ClearNew(new_bucket, new_bucket_i); + + old_target_info.nRefCount = 1; + vvNew[new_bucket][new_bucket_i] = old_target_id; + ++nNew; + } + + vvTried[bucket_target][i_target] = id; + vvTried[bucket][i] = -1; + addr_info = addr_info_newport; + } + } +} diff --git a/src/addrman.h b/src/addrman.h index 665e253192..2a5c6c06b4 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -450,6 +450,10 @@ public: LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); } + RemoveInvalid(); + + ResetI2PPorts(); + Check(); } @@ -762,6 +766,17 @@ private: //! Update an entry's service bits. void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + //! Remove invalid addresses. + void RemoveInvalid() EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** + * Reset the ports of I2P peers to 0. + * This is needed as a temporary measure because now we enforce port 0 and + * only connect to I2P hosts if the port is 0, but in the early days some + * I2P addresses with port 8333 were rumoured and persisted into addrmans. + */ + void ResetI2PPorts() EXCLUSIVE_LOCKS_REQUIRED(cs); + friend class CAddrManTest; }; diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp new file mode 100644 index 0000000000..0469f0cb4c --- /dev/null +++ b/src/bench/peer_eviction.cpp @@ -0,0 +1,156 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <net.h> +#include <netaddress.h> +#include <random.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> + +#include <algorithm> +#include <functional> +#include <vector> + +static void EvictionProtectionCommon( + benchmark::Bench& bench, + int num_candidates, + std::function<void(NodeEvictionCandidate&)> candidate_setup_fn) +{ + using Candidates = std::vector<NodeEvictionCandidate>; + FastRandomContext random_context{true}; + bench.warmup(100).epochIterations(1100); + + Candidates candidates{GetRandomNodeEvictionCandidates(num_candidates, random_context)}; + for (auto& c : candidates) { + candidate_setup_fn(c); + } + + std::vector<Candidates> copies{bench.epochs() * bench.epochIterations(), candidates}; + size_t i{0}; + bench.run([&] { + ProtectEvictionCandidatesByRatio(copies.at(i)); + ++i; + }); +} + +/* Benchmarks */ + +static void EvictionProtection0Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_network = NET_IPV4; + }); +} + +static void EvictionProtection1Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 130 && c.id < 240) { // 110 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection2Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 90 && c.id < 160) { // 70 Tor + c.m_network = NET_ONION; + } else if (c.id >= 170 && c.id < 250) { // 80 I2P + c.m_network = NET_I2P; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks050Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 50 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id == 28 || c.id == 47); // 2 localhost + if (c.id >= 30 && c.id < 47) { // 17 I2P + c.m_network = NET_I2P; + } else if (c.id >= 24 && c.id < 28) { // 4 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks100Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 100 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 55 && c.id < 60); // 5 localhost + if (c.id >= 70 && c.id < 80) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 80 && c.id < 96) { // 16 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 140 && c.id < 160); // 20 localhost + if (c.id >= 170 && c.id < 180) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 190 && c.id < 240) { // 50 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +// Candidate numbers used for the benchmarks: +// - 50 candidates simulates a possible use of -maxconnections +// - 100 candidates approximates an average node with default settings +// - 250 candidates is the number of peers reported by operators of busy nodes + +// No disadvantaged networks, with 250 eviction candidates. +BENCHMARK(EvictionProtection0Networks250Candidates); + +// 1 disadvantaged network (Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection1Networks250Candidates); + +// 2 disadvantaged networks (I2P, Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection2Networks250Candidates); + +// 3 disadvantaged networks (I2P/localhost/Tor) with 50/100/250 eviction candidates. +BENCHMARK(EvictionProtection3Networks050Candidates); +BENCHMARK(EvictionProtection3Networks100Candidates); +BENCHMARK(EvictionProtection3Networks250Candidates); diff --git a/src/chainparams.h b/src/chainparams.h index 5c2351eea6..4faa6f8d06 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -8,11 +8,13 @@ #include <chainparamsbase.h> #include <consensus/params.h> +#include <netaddress.h> #include <primitives/block.h> #include <protocol.h> #include <util/hash_type.h> #include <memory> +#include <string> #include <vector> typedef std::map<int, uint256> MapCheckpoints; @@ -80,6 +82,15 @@ public: const Consensus::Params& GetConsensus() const { return consensus; } const CMessageHeader::MessageStartChars& MessageStart() const { return pchMessageStart; } uint16_t GetDefaultPort() const { return nDefaultPort; } + uint16_t GetDefaultPort(Network net) const + { + return net == NET_I2P ? I2P_SAM31_PORT : GetDefaultPort(); + } + uint16_t GetDefaultPort(const std::string& addr) const + { + CNetAddr a; + return a.SetSpecial(addr) ? GetDefaultPort(a.GetNetwork()) : GetDefaultPort(); + } const CBlock& GenesisBlock() const { return genesis; } /** Default value for -checkmempool and -checkblockindex argument */ diff --git a/src/chainparamsseeds.h b/src/chainparamsseeds.h index 08587e2b37..a22529c386 100644 --- a/src/chainparamsseeds.h +++ b/src/chainparamsseeds.h @@ -683,14 +683,14 @@ static const uint8_t chainparams_seed_main[] = { 0x04,0x20,0x98,0xc6,0x44,0x27,0x90,0x41,0xa6,0x98,0xf9,0x25,0x6c,0x59,0x0f,0x06,0x6d,0x44,0x59,0x0e,0xb2,0x46,0xb0,0xa4,0x37,0x88,0x69,0x8f,0xc1,0x32,0xcd,0x9f,0x15,0xd7,0x20,0x8d, 0x04,0x20,0xaa,0x3a,0x16,0x86,0xea,0x59,0x09,0x04,0x78,0xe5,0x10,0x92,0xe1,0x1d,0xad,0xf7,0x56,0x2b,0xac,0xb0,0x97,0x29,0x63,0x30,0xf4,0x1b,0xcf,0xde,0xf3,0x28,0x0a,0x29,0x20,0x8d, 0x04,0x20,0xbc,0x27,0xae,0x89,0xc1,0x67,0x73,0x0a,0x08,0x02,0xdf,0xb7,0xcc,0x94,0xc7,0x9f,0xf4,0x72,0x7a,0x9b,0x20,0x0c,0x5c,0x11,0x3d,0x22,0xd6,0x13,0x88,0x66,0x74,0xbf,0x20,0x8d, - 0x05,0x20,0xfe,0x97,0xba,0x09,0x2a,0xa4,0x85,0x10,0xa1,0x04,0x7b,0x88,0x7a,0x5a,0x06,0x53,0x71,0x93,0x3b,0xf9,0xa2,0x2f,0xd9,0xe3,0x8f,0xa5,0xa2,0xac,0x1e,0x6c,0x6c,0x8c,0x20,0x8d, - 0x05,0x20,0x17,0x0c,0x56,0xce,0x72,0xa5,0xa0,0xe6,0x23,0x06,0xa3,0xc7,0x08,0x43,0x18,0xee,0x3a,0x46,0x35,0x5d,0x17,0xf6,0x78,0x96,0xa0,0x9c,0x51,0xef,0xbe,0x23,0xfd,0x71,0x20,0x8d, - 0x05,0x20,0x31,0x0f,0x30,0x0b,0x9d,0x70,0x0c,0x7c,0xf7,0x98,0x7e,0x1c,0xf4,0x33,0xdc,0x64,0x17,0xf7,0x00,0x7a,0x0c,0x04,0xb5,0x83,0xfc,0x5f,0xa6,0x52,0x39,0x79,0x63,0x87,0x20,0x8d, - 0x05,0x20,0x3e,0xe3,0xe0,0xa9,0xbc,0xf4,0x2e,0x59,0xd9,0x20,0xee,0xdf,0x74,0x61,0x4d,0x99,0x0c,0x5c,0x15,0x30,0x9b,0x72,0x16,0x79,0x15,0xf4,0x7a,0xca,0x34,0xcc,0x81,0x99,0x20,0x8d, - 0x05,0x20,0x3b,0x42,0x1c,0x25,0xf7,0xbf,0x79,0xed,0x6d,0x7d,0xef,0x65,0x30,0x7d,0xee,0x16,0x37,0x22,0x72,0x43,0x33,0x28,0x40,0xa3,0xaa,0xf4,0x48,0x49,0x67,0xb1,0x4b,0xfd,0x20,0x8d, - 0x05,0x20,0x7a,0x65,0xf7,0x47,0x42,0x9d,0x66,0x42,0x3b,0xb3,0xa7,0x03,0x6c,0x46,0x78,0x19,0x28,0x78,0x1e,0xa3,0x7c,0x67,0x44,0xb7,0x83,0x05,0xe3,0xfe,0xa5,0xe4,0x0a,0x6e,0x20,0x8d, - 0x05,0x20,0xb5,0x83,0x6f,0xb6,0x11,0xd8,0x0e,0xa8,0x57,0xda,0x15,0x20,0x5b,0x1a,0x6d,0x21,0x15,0x5a,0xbd,0xb4,0x17,0x11,0xc2,0xfb,0x0e,0xfc,0xde,0xe8,0x26,0x56,0xa8,0xac,0x20,0x8d, - 0x05,0x20,0xcc,0xaf,0x6c,0x3b,0xd0,0x13,0x76,0x23,0xc3,0x36,0xbb,0x64,0x4a,0x4a,0x06,0x93,0x69,0x6d,0xb0,0x10,0x6e,0x66,0xa4,0x61,0xf8,0x2d,0xe7,0x80,0x72,0x4d,0x53,0x94,0x20,0x8d, + 0x05,0x20,0xfe,0x97,0xba,0x09,0x2a,0xa4,0x85,0x10,0xa1,0x04,0x7b,0x88,0x7a,0x5a,0x06,0x53,0x71,0x93,0x3b,0xf9,0xa2,0x2f,0xd9,0xe3,0x8f,0xa5,0xa2,0xac,0x1e,0x6c,0x6c,0x8c,0x00,0x00, + 0x05,0x20,0x17,0x0c,0x56,0xce,0x72,0xa5,0xa0,0xe6,0x23,0x06,0xa3,0xc7,0x08,0x43,0x18,0xee,0x3a,0x46,0x35,0x5d,0x17,0xf6,0x78,0x96,0xa0,0x9c,0x51,0xef,0xbe,0x23,0xfd,0x71,0x00,0x00, + 0x05,0x20,0x31,0x0f,0x30,0x0b,0x9d,0x70,0x0c,0x7c,0xf7,0x98,0x7e,0x1c,0xf4,0x33,0xdc,0x64,0x17,0xf7,0x00,0x7a,0x0c,0x04,0xb5,0x83,0xfc,0x5f,0xa6,0x52,0x39,0x79,0x63,0x87,0x00,0x00, + 0x05,0x20,0x3e,0xe3,0xe0,0xa9,0xbc,0xf4,0x2e,0x59,0xd9,0x20,0xee,0xdf,0x74,0x61,0x4d,0x99,0x0c,0x5c,0x15,0x30,0x9b,0x72,0x16,0x79,0x15,0xf4,0x7a,0xca,0x34,0xcc,0x81,0x99,0x00,0x00, + 0x05,0x20,0x3b,0x42,0x1c,0x25,0xf7,0xbf,0x79,0xed,0x6d,0x7d,0xef,0x65,0x30,0x7d,0xee,0x16,0x37,0x22,0x72,0x43,0x33,0x28,0x40,0xa3,0xaa,0xf4,0x48,0x49,0x67,0xb1,0x4b,0xfd,0x00,0x00, + 0x05,0x20,0x7a,0x65,0xf7,0x47,0x42,0x9d,0x66,0x42,0x3b,0xb3,0xa7,0x03,0x6c,0x46,0x78,0x19,0x28,0x78,0x1e,0xa3,0x7c,0x67,0x44,0xb7,0x83,0x05,0xe3,0xfe,0xa5,0xe4,0x0a,0x6e,0x00,0x00, + 0x05,0x20,0xb5,0x83,0x6f,0xb6,0x11,0xd8,0x0e,0xa8,0x57,0xda,0x15,0x20,0x5b,0x1a,0x6d,0x21,0x15,0x5a,0xbd,0xb4,0x17,0x11,0xc2,0xfb,0x0e,0xfc,0xde,0xe8,0x26,0x56,0xa8,0xac,0x00,0x00, + 0x05,0x20,0xcc,0xaf,0x6c,0x3b,0xd0,0x13,0x76,0x23,0xc3,0x36,0xbb,0x64,0x4a,0x4a,0x06,0x93,0x69,0x6d,0xb0,0x10,0x6e,0x66,0xa4,0x61,0xf8,0x2d,0xe7,0x80,0x72,0x4d,0x53,0x94,0x00,0x00, }; static const uint8_t chainparams_seed_test[] = { diff --git a/src/consensus/params.h b/src/consensus/params.h index 174f4677fa..9205cfee87 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -11,8 +11,11 @@ namespace Consensus { -enum BuriedDeployment : int16_t -{ +/** + * A buried deployment is one where the height of the activation has been hardcoded into + * the client implementation long after the consensus change has activated. See BIP 90. + */ +enum BuriedDeployment : int16_t { // buried deployments get negative values to avoid overlap with DeploymentPos DEPLOYMENT_HEIGHTINCB = std::numeric_limits<int16_t>::min(), DEPLOYMENT_CLTV, @@ -22,8 +25,7 @@ enum BuriedDeployment : int16_t }; constexpr bool ValidDeployment(BuriedDeployment dep) { return DEPLOYMENT_HEIGHTINCB <= dep && dep <= DEPLOYMENT_SEGWIT; } -enum DeploymentPos : uint16_t -{ +enum DeploymentPos : uint16_t { DEPLOYMENT_TESTDUMMY, DEPLOYMENT_TAPROOT, // Deployment of Schnorr/Taproot (BIPs 340-342) // NOTE: Also add new deployments to VersionBitsDeploymentInfo in deploymentinfo.cpp diff --git a/src/deploymentstatus.h b/src/deploymentstatus.h index 84c5e54698..f95c5996f5 100644 --- a/src/deploymentstatus.h +++ b/src/deploymentstatus.h @@ -49,7 +49,7 @@ inline bool DeploymentEnabled(const Consensus::Params& params, Consensus::Buried inline bool DeploymentEnabled(const Consensus::Params& params, Consensus::DeploymentPos dep) { assert(Consensus::ValidDeployment(dep)); - return params.vDeployments[dep].nTimeout != 0; + return params.vDeployments[dep].nStartTime != Consensus::BIP9Deployment::NEVER_ACTIVE; } #endif // BITCOIN_DEPLOYMENTSTATUS_H diff --git a/src/i2p.cpp b/src/i2p.cpp index 2ae164633b..5e7e42fb77 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -159,7 +159,7 @@ bool Session::Accept(Connection& conn) const std::string& peer_dest = conn.sock->RecvUntilTerminator('\n', MAX_WAIT_FOR_IO, *m_interrupt, MAX_MSG_SIZE); - conn.peer = CService(DestB64ToAddr(peer_dest), Params().GetDefaultPort()); + conn.peer = CService(DestB64ToAddr(peer_dest), I2P_SAM31_PORT); return true; } @@ -172,6 +172,13 @@ bool Session::Accept(Connection& conn) bool Session::Connect(const CService& to, Connection& conn, bool& proxy_error) { + // Refuse connecting to arbitrary ports. We don't specify any destination port to the SAM proxy + // when connecting (SAM 3.1 does not use ports) and it forces/defaults it to I2P_SAM31_PORT. + if (to.GetPort() != I2P_SAM31_PORT) { + proxy_error = false; + return false; + } + proxy_error = true; std::string session_id; @@ -366,7 +373,7 @@ void Session::CreateIfNotCreatedAlready() SendRequestAndGetReply(*sock, strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=%s", session_id, private_key_b64)); - m_my_addr = CService(DestBinToAddr(MyDestination()), Params().GetDefaultPort()); + m_my_addr = CService(DestBinToAddr(MyDestination()), I2P_SAM31_PORT); m_session_id = session_id; m_control_sock = std::move(sock); diff --git a/src/init.cpp b/src/init.cpp index ae96f510bc..9afd76d62d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -442,7 +442,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections.", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections. Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1349,7 +1349,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - chainman.InitializeChainstate(*Assert(node.mempool)); + chainman.InitializeChainstate(Assert(node.mempool.get())); chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; @@ -1717,18 +1717,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(ResolveErrMsg("bind", bind_arg)); } - if (connOptions.onion_binds.empty()) { - connOptions.onion_binds.push_back(DefaultOnionServiceTarget()); - } - - if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { - const auto bind_addr = connOptions.onion_binds.front(); - if (connOptions.onion_binds.size() > 1) { - InitWarning(strprintf(_("More than one onion bind address is provided. Using %s for the automatically created Tor onion service."), bind_addr.ToStringIPPort())); - } - StartTorControl(bind_addr); - } - for (const std::string& strBind : args.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; bilingual_str error; @@ -1736,6 +1724,27 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.vWhiteBinds.push_back(whitebind); } + // If the user did not specify -bind= or -whitebind= then we bind + // on any address - 0.0.0.0 (IPv4) and :: (IPv6). + connOptions.bind_on_any = args.GetArgs("-bind").empty() && args.GetArgs("-whitebind").empty(); + + CService onion_service_target; + if (!connOptions.onion_binds.empty()) { + onion_service_target = connOptions.onion_binds.front(); + } else { + onion_service_target = DefaultOnionServiceTarget(); + connOptions.onion_binds.push_back(onion_service_target); + } + + if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { + if (connOptions.onion_binds.size() > 1) { + InitWarning(strprintf(_("More than one onion bind address is provided. Using %s " + "for the automatically created Tor onion service."), + onion_service_target.ToStringIPPort())); + } + StartTorControl(onion_service_target); + } + for (const auto& net : args.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; bilingual_str error; diff --git a/src/init.h b/src/init.h index b856468e5d..5af6930a16 100644 --- a/src/init.h +++ b/src/init.h @@ -20,9 +20,6 @@ struct NodeContext; namespace interfaces { struct BlockAndHeaderTipInfo; } -namespace boost { -class thread_group; -} // namespace boost /** Interrupt threads */ void Interrupt(NodeContext& node); diff --git a/src/net.cpp b/src/net.cpp index 60059249ed..70ba875c4b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -402,7 +402,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo pszDest ? 0.0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); // Resolve - const uint16_t default_port{Params().GetDefaultPort()}; + const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : + Params().GetDefaultPort()}; if (pszDest) { std::vector<CService> resolved; if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { @@ -936,14 +937,17 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti size_t num_protected{0}; while (num_protected < max_protect_by_network) { + // Count the number of disadvantaged networks from which we have peers to protect. + auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); + if (num_networks == 0) { + break; + } const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; - const size_t protect_per_network{ - std::max(disadvantaged_to_protect / networks.size(), static_cast<size_t>(1))}; - + const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; // Early exit flag if there are no remaining candidates by disadvantaged network. bool protected_at_least_one{false}; - for (const Net& n : networks) { + for (Net& n : networks) { if (n.count == 0) continue; const size_t before = eviction_candidates.size(); EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), @@ -953,10 +957,12 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti const size_t after = eviction_candidates.size(); if (before > after) { protected_at_least_one = true; - num_protected += before - after; + const size_t delta{before - after}; + num_protected += delta; if (num_protected >= max_protect_by_network) { break; } + n.count -= delta; } } if (!protected_at_least_one) { @@ -2059,8 +2065,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // from advertising themselves as a service on another host and // port, causing a DoS attack as nodes around the network attempt // to connect to it fruitlessly. - if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) + if (addr.GetPort() != Params().GetDefaultPort(addr.GetNetwork()) && nTries < 50) { continue; + } addrConnect = addr; break; @@ -2123,7 +2130,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const } for (const std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); + CService service(LookupNumeric(strAddNode, Params().GetDefaultPort(strAddNode))); AddedNodeInfo addedNode{strAddNode, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port @@ -2465,30 +2472,25 @@ bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags return true; } -bool CConnman::InitBinds( - const std::vector<CService>& binds, - const std::vector<NetWhitebindPermissions>& whiteBinds, - const std::vector<CService>& onion_binds) +bool CConnman::InitBinds(const Options& options) { bool fBound = false; - for (const auto& addrBind : binds) { + for (const auto& addrBind : options.vBinds) { fBound |= Bind(addrBind, (BF_EXPLICIT | BF_REPORT_ERROR), NetPermissionFlags::None); } - for (const auto& addrBind : whiteBinds) { + for (const auto& addrBind : options.vWhiteBinds) { fBound |= Bind(addrBind.m_service, (BF_EXPLICIT | BF_REPORT_ERROR), addrBind.m_flags); } - if (binds.empty() && whiteBinds.empty()) { + for (const auto& addr_bind : options.onion_binds) { + fBound |= Bind(addr_bind, BF_EXPLICIT | BF_DONT_ADVERTISE, NetPermissionFlags::None); + } + if (options.bind_on_any) { struct in_addr inaddr_any; inaddr_any.s_addr = htonl(INADDR_ANY); struct in6_addr inaddr6_any = IN6ADDR_ANY_INIT; fBound |= Bind(CService(inaddr6_any, GetListenPort()), BF_NONE, NetPermissionFlags::None); fBound |= Bind(CService(inaddr_any, GetListenPort()), !fBound ? BF_REPORT_ERROR : BF_NONE, NetPermissionFlags::None); } - - for (const auto& addr_bind : onion_binds) { - fBound |= Bind(addr_bind, BF_EXPLICIT | BF_DONT_ADVERTISE, NetPermissionFlags::None); - } - return fBound; } @@ -2496,7 +2498,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { Init(connOptions); - if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds, connOptions.onion_binds)) { + if (fListen && !InitBinds(connOptions)) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( _("Failed to listen on any port. Use -listen=0 if you want this."), @@ -768,6 +768,9 @@ public: std::vector<NetWhitebindPermissions> vWhiteBinds; std::vector<CService> vBinds; std::vector<CService> onion_binds; + /// True if the user did not specify -bind= or -whitebind= and thus + /// we should bind on `0.0.0.0` (IPv4) and `::` (IPv6). + bool bind_on_any; bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; @@ -962,10 +965,7 @@ private: bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); - bool InitBinds( - const std::vector<CService>& binds, - const std::vector<NetWhitebindPermissions>& whiteBinds, - const std::vector<CService>& onion_binds); + bool InitBinds(const Options& options); void ThreadOpenAddedConnections(); void AddAddrFetch(const std::string& strDest); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 1ea3969978..e7b3377475 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -489,7 +489,7 @@ bool CNetAddr::IsValid() const */ bool CNetAddr::IsRoutable() const { - return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); + return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || IsRFC4193() || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); } /** diff --git a/src/netaddress.h b/src/netaddress.h index dd47ab5749..d504996347 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -112,6 +112,9 @@ static constexpr size_t ADDR_CJDNS_SIZE = 16; /// Size of "internal" (NET_INTERNAL) address (in bytes). static constexpr size_t ADDR_INTERNAL_SIZE = 10; +/// SAM 3.1 and earlier do not support specifying ports and force the port to 0. +static constexpr uint16_t I2P_SAM31_PORT{0}; + /** * Network address. */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b630458f23..c4a89c9772 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1350,10 +1350,7 @@ static RPCHelpMan verifychain() static void SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& softforks, const Consensus::Params& params, Consensus::BuriedDeployment dep) { // For buried deployments. - // A buried deployment is one where the height of the activation has been hardcoded into - // the client implementation long after the consensus change has activated. See BIP 90. - // Buried deployments with activation height value of - // std::numeric_limits<int>::max() are disabled and thus hidden. + if (!DeploymentEnabled(params, dep)) return; UniValue rv(UniValue::VOBJ); @@ -1368,8 +1365,8 @@ static void SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& static void SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& softforks, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) { // For BIP9 deployments. - // Deployments that are never active are hidden. - if (consensusParams.vDeployments[id].nStartTime == Consensus::BIP9Deployment::NEVER_ACTIVE) return; + + if (!DeploymentEnabled(consensusParams, id)) return; UniValue bip9(UniValue::VOBJ); const ThresholdState thresholdState = g_versionbitscache.State(active_chain_tip, consensusParams, id); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ccb3123714..617dfec98f 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -894,8 +894,7 @@ static RPCHelpMan testmempoolaccept() "\nThis checks if transactions violate the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", { - {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.\n" - " Length must be one for now.", + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", { {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index eb5c37b34d..f2eed956cd 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <i2p.h> #include <test/data/asmap.raw.h> #include <test/util/setup_common.h> #include <util/asmap.h> @@ -783,6 +784,46 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); } +BOOST_AUTO_TEST_CASE(remove_invalid) +{ + // Confirm that invalid addresses are ignored in unserialization. + + CAddrManTest addrman; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; + const CAddress new2{ResolveService("6.6.6.6"), NODE_NONE}; + const CAddress tried1{ResolveService("7.7.7.7"), NODE_NONE}; + const CAddress tried2{ResolveService("8.8.8.8"), NODE_NONE}; + + addrman.Add({new1, tried1, new2, tried2}, CNetAddr{}); + addrman.Good(tried1); + addrman.Good(tried2); + BOOST_REQUIRE_EQUAL(addrman.size(), 4); + + stream << addrman; + + const std::string str{stream.str()}; + size_t pos; + + const char new2_raw[]{6, 6, 6, 6}; + const uint8_t new2_raw_replacement[]{0, 0, 0, 0}; // 0.0.0.0 is !IsValid() + pos = str.find(new2_raw, 0, sizeof(new2_raw)); + BOOST_REQUIRE(pos != std::string::npos); + BOOST_REQUIRE(pos + sizeof(new2_raw_replacement) <= stream.size()); + memcpy(stream.data() + pos, new2_raw_replacement, sizeof(new2_raw_replacement)); + + const char tried2_raw[]{8, 8, 8, 8}; + const uint8_t tried2_raw_replacement[]{255, 255, 255, 255}; // 255.255.255.255 is !IsValid() + pos = str.find(tried2_raw, 0, sizeof(tried2_raw)); + BOOST_REQUIRE(pos != std::string::npos); + BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); + memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); + + addrman.Clear(); + stream >> addrman; + BOOST_CHECK_EQUAL(addrman.size(), 2); +} BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { @@ -926,5 +967,121 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); } +BOOST_AUTO_TEST_CASE(reset_i2p_ports) +{ + CAddrManTest addrman1; + CAddrManTest addrman2; + const uint32_t good_time{static_cast<uint32_t>(GetAdjustedTime())}; + constexpr uint16_t port = 8333; + + // Has its port changed, will be re-positioned within the same bucket in vvNew. + const CAddress i2p_new1{ + ResolveService("72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p", port), + NODE_NONE, + good_time}; + + // Has its port changed, will not be re-positioned in vvNew because ports 0 and 10075 result in + // the same bucket position. + const CAddress i2p_new2{ + ResolveService("gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p", 10075), + NODE_NONE, + good_time}; + + // Remains unchanged, port is already as it should be. + const CAddress i2p_new3{ + ResolveService("c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p", + I2P_SAM31_PORT), + NODE_NONE, + good_time}; + + // Has its port changed, re-positioning in vvNew will cause i2p_new3 to be evicted. + const CAddress i2p_new4{ + ResolveService("c4cbbkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p", port), + NODE_NONE, + good_time}; + + // Remains unchanged. + const CAddress ipv4_new{ResolveService("1.2.3.4", port), NODE_NONE, good_time}; + + // Has its port changed, will be re-positioned in vvTried. + const CAddress i2p_tried1{ + ResolveService("h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p", port), + NODE_NONE, + good_time}; + + // Has its port changed, will not be re-positioned in vvTried because ports 0 and 10537 + // result in the same position (bucket, i). + const CAddress i2p_tried2{ + ResolveService("pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p", 10537), + NODE_NONE, + good_time}; + + // Remains unchanged, port is already as it should be. + const CAddress i2p_tried3{ + ResolveService("hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p", + I2P_SAM31_PORT), + NODE_NONE, + good_time}; + + // Has its port changed, re-positioning in vvTried will cause i2p_tried3 to be moved to vvNew. + const CAddress i2p_tried4{ + ResolveService("hna37nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p", port), + NODE_NONE, + good_time}; + + // Remains unchanged. + const CAddress ipv4_tried{ResolveService("2.3.4.5", port), NODE_NONE, good_time}; + + const CNetAddr source; + + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + addrman1.Add(i2p_new1, source); + addrman1.Add(i2p_new2, source); + addrman1.Add(i2p_new3, source); + addrman1.Add(i2p_new4, source); + addrman1.Add(ipv4_new, source); + + addrman1.Add(i2p_tried1, source); + addrman1.Good(i2p_tried1); + addrman1.Add(i2p_tried2, source); + addrman1.Good(i2p_tried2); + addrman1.Add(i2p_tried3, source); + addrman1.Good(i2p_tried3); + addrman1.Add(i2p_tried4, source); + addrman1.Good(i2p_tried4); + addrman1.Add(ipv4_tried, source); + addrman1.Good(ipv4_tried); + + stream << addrman1; + stream >> addrman2; + + const size_t max_addresses{0}; + const size_t max_pct{0}; + + auto addresses = addrman2.GetAddr(max_addresses, max_pct, NET_I2P); + BOOST_REQUIRE_EQUAL(addresses.size(), 7UL); + std::sort(addresses.begin(), addresses.end()); // Just some deterministic order. + BOOST_CHECK_EQUAL(addresses[0].ToStringIP(), i2p_new4.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[0].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[1].ToStringIP(), i2p_new2.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[1].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[2].ToStringIP(), i2p_tried4.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[2].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[3].ToStringIP(), i2p_tried3.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[3].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[4].ToStringIP(), i2p_tried1.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[4].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[5].ToStringIP(), i2p_tried2.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[5].GetPort(), I2P_SAM31_PORT); + BOOST_CHECK_EQUAL(addresses[6].ToStringIP(), i2p_new1.ToStringIP()); + BOOST_CHECK_EQUAL(addresses[6].GetPort(), I2P_SAM31_PORT); + + addresses = addrman2.GetAddr(max_addresses, max_pct, NET_IPV4); + BOOST_REQUIRE_EQUAL(addresses.size(), 2UL); + std::sort(addresses.begin(), addresses.end()); // Just some deterministic order. + BOOST_CHECK_EQUAL(addresses[0].ToStringIPPort(), ipv4_new.ToStringIPPort()); + BOOST_CHECK_EQUAL(addresses[1].ToStringIPPort(), ipv4_tried.ToStringIPPort()); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index e9fa343896..e28e2feb0a 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -16,8 +16,6 @@ #include <pow.h> #include <protocol.h> #include <pubkey.h> -#include <rpc/util.h> -#include <script/signingprovider.h> #include <script/standard.h> #include <serialize.h> #include <streams.h> @@ -158,20 +156,6 @@ FUZZ_TARGET_INIT(integer, initialize_integer) const CKeyID key_id{u160}; const CScriptID script_id{u160}; - // CTxDestination = CNoDestination ∪ PKHash ∪ ScriptHash ∪ WitnessV0ScriptHash ∪ WitnessV0KeyHash ∪ WitnessUnknown - const PKHash pk_hash{u160}; - const ScriptHash script_hash{u160}; - const WitnessV0KeyHash witness_v0_key_hash{u160}; - const WitnessV0ScriptHash witness_v0_script_hash{u256}; - const std::vector<CTxDestination> destinations{pk_hash, script_hash, witness_v0_key_hash, witness_v0_script_hash}; - const SigningProvider store; - for (const CTxDestination& destination : destinations) { - (void)DescribeAddress(destination); - (void)EncodeDestination(destination); - (void)GetKeyForDestination(store, destination); - (void)GetScriptForDestination(destination); - (void)IsValidDestination(destination); - } { CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 665ca01fa1..f58bf8b316 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -4,9 +4,6 @@ #include <chainparams.h> #include <key_io.h> -#include <rpc/util.h> -#include <script/signingprovider.h> -#include <script/standard.h> #include <test/fuzz/fuzz.h> #include <cassert> @@ -39,12 +36,4 @@ FUZZ_TARGET_INIT(key_io, initialize_key_io) if (ext_pub_key.pubkey.size() == CPubKey::COMPRESSED_SIZE) { assert(ext_pub_key == DecodeExtPubKey(EncodeExtPubKey(ext_pub_key))); } - - const CTxDestination tx_destination = DecodeDestination(random_string); - (void)DescribeAddress(tx_destination); - (void)GetKeyForDestination(/* store */ {}, tx_destination); - (void)GetScriptForDestination(tx_destination); - (void)IsValidDestination(tx_destination); - - (void)IsValidDestinationString(random_string); } diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index f9d8129ca9..6cb81901cb 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -54,7 +54,7 @@ FUZZ_TARGET(netaddress) (void)net_addr.IsRFC3927(); (void)net_addr.IsRFC3964(); if (net_addr.IsRFC4193()) { - assert(net_addr.GetNetwork() == Network::NET_ONION || net_addr.GetNetwork() == Network::NET_INTERNAL || net_addr.GetNetwork() == Network::NET_UNROUTABLE); + assert(net_addr.GetNetwork() == Network::NET_INTERNAL || net_addr.GetNetwork() == Network::NET_UNROUTABLE); } (void)net_addr.IsRFC4380(); (void)net_addr.IsRFC4843(); diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index b87bcf2ef5..950ee45d1d 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -6,8 +6,10 @@ #include <compressor.h> #include <core_io.h> #include <core_memusage.h> +#include <key_io.h> #include <policy/policy.h> #include <pubkey.h> +#include <rpc/util.h> #include <script/descriptor.h> #include <script/interpreter.h> #include <script/script.h> @@ -184,26 +186,26 @@ FUZZ_TARGET_INIT(script, initialize_script) } { - WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown_1.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); - - WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown_2.length = witness_unknown_program_2.size(); - std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); - - (void)(witness_unknown_1 == witness_unknown_2); - (void)(witness_unknown_1 < witness_unknown_2); - } + const CTxDestination tx_destination_1{ + fuzzed_data_provider.ConsumeBool() ? + DecodeDestination(fuzzed_data_provider.ConsumeRandomLengthString()) : + ConsumeTxDestination(fuzzed_data_provider)}; + const CTxDestination tx_destination_2{ConsumeTxDestination(fuzzed_data_provider)}; + const std::string encoded_dest{EncodeDestination(tx_destination_1)}; + const UniValue json_dest{DescribeAddress(tx_destination_1)}; + Assert(tx_destination_1 == DecodeDestination(encoded_dest)); + (void)GetKeyForDestination(/* store */ {}, tx_destination_1); + const CScript dest{GetScriptForDestination(tx_destination_1)}; + const bool valid{IsValidDestination(tx_destination_1)}; + Assert(dest.empty() != valid); + + Assert(valid == IsValidDestinationString(encoded_dest)); - { - const CTxDestination tx_destination_1 = ConsumeTxDestination(fuzzed_data_provider); - const CTxDestination tx_destination_2 = ConsumeTxDestination(fuzzed_data_provider); - (void)(tx_destination_1 == tx_destination_2); (void)(tx_destination_1 < tx_destination_2); + if (tx_destination_1 == tx_destination_2) { + Assert(encoded_dest == EncodeDestination(tx_destination_2)); + Assert(json_dest.write() == DescribeAddress(tx_destination_2).write()); + Assert(dest == GetScriptForDestination(tx_destination_2)); + } } } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index bcf0b0ce72..ece3214ed5 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> #include <util/rbf.h> @@ -304,3 +305,41 @@ uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept }) : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); } + +CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + CTxDestination tx_destination; + const size_t call_size{CallOneOf( + fuzzed_data_provider, + [&] { + tx_destination = CNoDestination{}; + }, + [&] { + tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV1Taproot{XOnlyPubKey{ConsumeUInt256(fuzzed_data_provider)}}; + }, + [&] { + WitnessUnknown witness_unknown{}; + witness_unknown.version = fuzzed_data_provider.ConsumeIntegralInRange(2, 16); + std::vector<uint8_t> witness_unknown_program_1{fuzzed_data_provider.ConsumeBytes<uint8_t>(40)}; + if (witness_unknown_program_1.size() < 2) { + witness_unknown_program_1 = {0, 0}; + } + witness_unknown.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); + tx_destination = witness_unknown; + })}; + Assert(call_size == std::variant_size_v<CTxDestination>); + return tx_destination; +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 023dcdb3e5..9f09395a9a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -37,7 +37,7 @@ #include <vector> template <typename... Callables> -void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) +size_t CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) { constexpr size_t call_size{sizeof...(callables)}; static_assert(call_size >= 1); @@ -45,6 +45,7 @@ void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) size_t i{0}; ((i++ == call_index ? callables() : void()), ...); + return call_size; } template <typename Collection> @@ -178,36 +179,7 @@ template <typename WeakEnumType, size_t size> return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; } -[[nodiscard]] inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - CTxDestination tx_destination; - CallOneOf( - fuzzed_data_provider, - [&] { - tx_destination = CNoDestination{}; - }, - [&] { - tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); - tx_destination = witness_unknown; - }); - return tx_destination; -} +[[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; template <typename T> [[nodiscard]] bool MultiplicationOverflow(const T i, const T j) noexcept diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 4bfd487b86..5eb280b498 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -17,28 +17,6 @@ BOOST_FIXTURE_TEST_SUITE(net_peer_eviction_tests, BasicTestingSetup) -std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_candidates, FastRandomContext& random_context) -{ - std::vector<NodeEvictionCandidate> candidates; - for (int id = 0; id < n_candidates; ++id) { - candidates.push_back({ - /* id */ id, - /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, - /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), - /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), - /* fRelevantServices */ random_context.randbool(), - /* fRelayTxes */ random_context.randbool(), - /* fBloomFilter */ random_context.randbool(), - /* nKeyedNetGroup */ random_context.randrange(100), - /* prefer_evict */ random_context.randbool(), - /* m_is_local */ random_context.randbool(), - /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], - }); - } - return candidates; -} - // Create `num_peers` random nodes, apply setup function `candidate_setup_fn`, // call ProtectEvictionCandidatesByRatio() to apply protection logic, and then // return true if all of `protected_peer_ids` and none of `unprotected_peer_ids` diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp index 54e07b0f61..7876c0bcda 100644 --- a/src/test/serfloat_tests.cpp +++ b/src/test/serfloat_tests.cpp @@ -36,9 +36,9 @@ uint64_t TestDouble(double f) { } // namespace BOOST_AUTO_TEST_CASE(double_serfloat_tests) { - BOOST_CHECK_EQUAL(TestDouble(0.0), 0); + BOOST_CHECK_EQUAL(TestDouble(0.0), 0U); BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000); - BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000U); BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::infinity()), 0xfff0000000000000); BOOST_CHECK_EQUAL(TestDouble(0.5), 0x3fe0000000000000ULL); BOOST_CHECK_EQUAL(TestDouble(1.0), 0x3ff0000000000000ULL); @@ -48,8 +48,8 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { // Roundtrip test on IEC559-compatible systems if (std::numeric_limits<double>::is_iec559) { - BOOST_CHECK_EQUAL(sizeof(double), 8); - BOOST_CHECK_EQUAL(sizeof(uint64_t), 8); + BOOST_CHECK_EQUAL(sizeof(double), 8U); + BOOST_CHECK_EQUAL(sizeof(uint64_t), 8U); // Test extreme values TestDouble(std::numeric_limits<double>::min()); TestDouble(-std::numeric_limits<double>::min()); diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 7af2b79f37..acd0151e1a 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -119,7 +119,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) uint32_t varint = 0; // Deserialize into r-value reader >> VARINT(varint); - BOOST_CHECK_EQUAL(varint, 54321); + BOOST_CHECK_EQUAL(varint, 54321U); BOOST_CHECK(reader.empty()); } diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 847a490e03..28d7967078 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -6,6 +6,9 @@ #include <chainparams.h> #include <net.h> +#include <span.h> + +#include <vector> void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const { @@ -37,3 +40,25 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) con NodeReceiveMsgBytes(node, ser_msg.data, complete); return complete; } + +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) +{ + std::vector<NodeEvictionCandidate> candidates; + for (int id = 0; id < n_candidates; ++id) { + candidates.push_back({ + /* id */ id, + /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), + /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, + /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), + /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), + /* fRelevantServices */ random_context.randbool(), + /* fRelayTxes */ random_context.randbool(), + /* fBloomFilter */ random_context.randbool(), + /* nKeyedNetGroup */ random_context.randrange(100), + /* prefer_evict */ random_context.randbool(), + /* m_is_local */ random_context.randbool(), + /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + }); + } + return candidates; +} diff --git a/src/test/util/net.h b/src/test/util/net.h index 1b49a671bd..939ec322ed 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -141,4 +141,6 @@ private: mutable size_t m_consumed; }; +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context); + #endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 748272bb1d..d9d236be1d 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -180,7 +180,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - m_node.chainman->InitializeChainstate(*m_node.mempool); + m_node.chainman->InitializeChainstate(m_node.mempool.get()); m_node.chainman->ActiveChainstate().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!m_node.chainman->ActiveChainstate().CanFlushToDisk()); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 92d8cf2e7d..2893b412fb 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 7c1db9d4b9..0bd378631b 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // const uint256 snapshot_blockhash = GetRandHash(); CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( - mempool, snapshot_blockhash)); + &mempool, snapshot_blockhash)); chainstates.push_back(&c2); BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -147,7 +147,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index a3b344d2c9..22aafcaa6c 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -20,10 +20,9 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate{mempool, blockman}; + CChainState chainstate{&mempool, blockman}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); - CTxMemPool tx_pool{}; constexpr bool is_64_bit = sizeof(void*) == 8; @@ -57,7 +56,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common @@ -72,7 +71,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); @@ -93,7 +92,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); } @@ -101,26 +100,26 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); - if (chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == + if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); } @@ -136,7 +135,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BOOST_CHECK(usage_percentage >= 0.9); BOOST_CHECK(usage_percentage < 1); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 1 << 10), CoinsCacheSizeState::LARGE); } @@ -144,7 +143,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 1000; ++i) { add_coin(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool), + chainstate.GetCoinsCacheSizeState(), CoinsCacheSizeState::OK); } @@ -152,7 +151,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // preallocated memory that doesn't get reclaimed even after flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); view.SetBestBlock(InsecureRand256()); @@ -160,7 +159,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); } diff --git a/src/validation.cpp b/src/validation.cpp index 65d2dfa3b7..26333d7026 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -329,23 +329,14 @@ static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_ return true; } -/* Make mempool consistent after a reorg, by re-adding or recursively erasing - * disconnected block transactions from the mempool, and also removing any - * other transactions from the mempool that are no longer valid given the new - * tip/height. - * - * Note: we assume that disconnectpool only contains transactions that are NOT - * confirmed in the current chain nor already in the mempool (otherwise, - * in-mempool descendants of such transactions would be removed). - * - * Passing fAddToMempool=false will skip trying to add the transactions back, - * and instead just erase from the mempool as needed. - */ - -static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) +void CChainState::MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) { + if (!m_mempool) return; + AssertLockHeld(cs_main); - AssertLockHeld(mempool.cs); + AssertLockHeld(m_mempool->cs); std::vector<uint256> vHashUpdate; // disconnectpool's insertion_order index sorts the entries from // oldest to newest, but the oldest entry will be the last tx from the @@ -357,11 +348,13 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions if (!fAddToMempool || (*it)->IsCoinBase() || - AcceptToMemoryPool(active_chainstate, mempool, *it, true /* bypass_limits */).m_result_type != MempoolAcceptResult::ResultType::VALID) { + AcceptToMemoryPool( + *this, *m_mempool, *it, true /* bypass_limits */).m_result_type != + MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). - mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (mempool.exists((*it)->GetHash())) { + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (m_mempool->exists((*it)->GetHash())) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -372,12 +365,16 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in // the disconnectpool that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); + m_mempool->UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(active_chainstate, STANDARD_LOCKTIME_VERIFY_FLAGS); + m_mempool->removeForReorg(*this, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions - LimitMempoolSize(mempool, active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + LimitMempoolSize( + *m_mempool, + this->CoinsTip(), + gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, + std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } /** @@ -587,9 +584,13 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); - // is it already in the memory pool? - if (m_pool.exists(hash)) { + if (m_pool.exists(GenTxid(true, tx.GetWitnessHash()))) { + // Exact transaction already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); + } else if (m_pool.exists(GenTxid(false, tx.GetHash()))) { + // Transaction with the same non-witness data but different witness (same txid, different + // wtxid) already exists in the mempool. + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-same-nonwitness-data-in-mempool"); } // Check for conflicts with in-memory transactions @@ -1208,7 +1209,7 @@ void CoinsViews::InitCache() m_cacheview = std::make_unique<CCoinsViewCache>(&m_catcherview); } -CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) +CChainState::CChainState(CTxMemPool* mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), m_params(::Params()), m_blockman(blockman), @@ -2001,20 +2002,18 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return true; } -CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool* tx_pool) +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState() { return this->GetCoinsCacheSizeState( - tx_pool, m_coinstip_cache_size_bytes, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) { - const int64_t nMempoolUsage = tx_pool ? tx_pool->DynamicMemoryUsage() : 0; + const int64_t nMempoolUsage = m_mempool ? m_mempool->DynamicMemoryUsage() : 0; int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); int64_t nTotalSpace = max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); @@ -2053,7 +2052,7 @@ bool CChainState::FlushStateToDisk( bool fFlushForPrune = false; bool fDoFullFlush = false; - CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above the blockfilterindexes bestblocks @@ -2204,12 +2203,12 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } -/** Check warning conditions and do some notifications on new chain tip set. */ -static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const CChainParams& chainParams, CChainState& active_chainstate) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +void CChainState::UpdateTip(const CBlockIndex* pindexNew) { // New best block - mempool.AddTransactionsUpdated(1); + if (m_mempool) { + m_mempool->AddTransactionsUpdated(1); + } { LOCK(g_best_block_mutex); @@ -2218,11 +2217,11 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } bilingual_str warning_messages; - if (!active_chainstate.IsInitialBlockDownload()) { + if (!this->IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); - ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); + ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache[bit]); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2237,14 +2236,14 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), pindexNew), active_chainstate.CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), active_chainstate.CoinsTip().GetCacheSize(), + GuessVerificationProgress(m_params.TxData(), pindexNew), this->CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), this->CoinsTip().GetCacheSize(), !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); } /** Disconnect m_chain's tip. * After calling, the mempool will be in an inconsistent state, with * transactions from disconnected blocks being added to disconnectpool. You - * should make the mempool consistent again by calling UpdateMempoolForReorg. + * should make the mempool consistent again by calling MaybeUpdateMempoolForReorg. * with cs_main held. * * If disconnectpool is nullptr, then no disconnected transactions are added to @@ -2254,7 +2253,7 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); @@ -2280,7 +2279,7 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr return false; } - if (disconnectpool) { + if (disconnectpool && m_mempool) { // Save transactions to re-add to mempool at end of reorg for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { disconnectpool->addTransaction(*it); @@ -2288,14 +2287,14 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { // Drop the earliest entry, and remove its children from the mempool. auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); - m_mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); disconnectpool->removeEntry(it); } } m_chain.SetTip(pindexDelete->pprev); - UpdateTip(m_mempool, pindexDelete->pprev, m_params, *this); + UpdateTip(pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock, pindexDelete); @@ -2357,7 +2356,7 @@ public: bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. @@ -2401,11 +2400,13 @@ bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); // Remove conflicting transactions from the mempool.; - m_mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); - disconnectpool.removeForBlock(blockConnecting.vtx); + if (m_mempool) { + m_mempool->removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); + } // Update m_chain & related variables. m_chain.SetTip(pindexNew); - UpdateTip(m_mempool, pindexNew, m_params, *this); + UpdateTip(pindexNew); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); @@ -2495,7 +2496,7 @@ void CChainState::PruneBlockIndexCandidates() { bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); const CBlockIndex* pindexOldTip = m_chain.Tip(); const CBlockIndex* pindexFork = m_chain.FindFork(pindexMostWork); @@ -2507,7 +2508,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex if (!DisconnectTip(state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort @@ -2551,7 +2552,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex // A system error occurred (disk space, database error, ...). // Make the mempool consistent with the current tip, just in case // any observers try to use it before shutdown. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2568,9 +2569,9 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, true); + MaybeUpdateMempoolForReorg(disconnectpool, true); } - m_mempool.check(*this); + if (m_mempool) m_mempool->check(*this); CheckForkWarningConditions(); @@ -2642,7 +2643,8 @@ bool CChainState::ActivateBestChain(BlockValidationState& state, std::shared_ptr { LOCK(cs_main); - LOCK(m_mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + LOCK(MempoolMutex()); CBlockIndex* starting_tip = m_chain.Tip(); bool blocks_connected = false; do { @@ -2792,7 +2794,9 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind LimitValidationInterfaceQueue(); LOCK(cs_main); - LOCK(m_mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between + // Lock for as long as disconnectpool is in scope to make sure MaybeUpdateMempoolForReorg is + // called after DisconnectTip without unlocking in between + LOCK(MempoolMutex()); if (!m_chain.Contains(pindex)) break; pindex_was_in_chain = true; CBlockIndex *invalid_walk_tip = m_chain.Tip(); @@ -2806,7 +2810,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pind // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + MaybeUpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); if (!ret) return false; assert(invalid_walk_tip->pprev == m_chain.Tip()); @@ -3817,10 +3821,11 @@ bool CChainState::LoadBlockIndexDB() void CChainState::LoadMempool(const ArgsManager& args) { + if (!m_mempool) return; if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(m_mempool, *this); + ::LoadMempool(*m_mempool, *this); } - m_mempool.SetIsLoaded(!ShutdownRequested()); + m_mempool->SetIsLoaded(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4684,7 +4689,8 @@ std::vector<CChainState*> ChainstateManager::GetAll() return out; } -CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash) +CChainState& ChainstateManager::InitializeChainstate( + CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) { bool is_snapshot = snapshot_blockhash.has_value(); std::unique_ptr<CChainState>& to_modify = @@ -4763,7 +4769,7 @@ bool ChainstateManager::ActivateSnapshot( } auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique<CChainState>( - this->ActiveChainstate().m_mempool, m_blockman, base_blockhash)); + /* mempool */ nullptr, m_blockman, base_blockhash)); { LOCK(::cs_main); @@ -4879,7 +4885,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } const auto snapshot_cache_state = WITH_LOCK(::cs_main, - return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool)); + return snapshot_chainstate.GetCoinsCacheSizeState()); if (snapshot_cache_state >= CoinsCacheSizeState::CRITICAL) { diff --git a/src/validation.h b/src/validation.h index 3d66e3161d..9a2be3ad97 100644 --- a/src/validation.h +++ b/src/validation.h @@ -587,8 +587,9 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! mempool that is kept in sync with the chain - CTxMemPool& m_mempool; + //! Optional mempool that is kept in sync with the chain. + //! Only the active chainstate has a mempool. + CTxMemPool* m_mempool; const CChainParams& m_params; @@ -600,7 +601,10 @@ public: //! CChainState instances. BlockManager& m_blockman; - explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); + explicit CChainState( + CTxMemPool* mempool, + BlockManager& blockman, + std::optional<uint256> from_snapshot_blockhash = std::nullopt); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -729,7 +733,7 @@ public: CCoinsViewCache& view, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); // Manual block validity manipulation: /** Mark a block as precious and reorganize. @@ -773,19 +777,17 @@ public: //! Dictates whether we need to flush the cache to disk or not. //! //! @return the state of the size of the coins cache. - CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool* tx_pool) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CoinsCacheSizeState GetCoinsCacheSizeState() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CoinsCacheSizeState GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); private: - bool ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); - bool ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + bool ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); void InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -798,6 +800,33 @@ private: bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Indirection necessary to make lock annotations work with an optional mempool. + RecursiveMutex* MempoolMutex() const LOCK_RETURNED(m_mempool->cs) + { + return m_mempool ? &m_mempool->cs : nullptr; + } + + /** + * Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + void MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + + /** Check warning conditions and do some notifications on new chain tip set. */ + void UpdateTip(const CBlockIndex* pindexNew) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + friend ChainstateManager; }; @@ -907,7 +936,9 @@ public: // constructor //! @param[in] snapshot_blockhash If given, signify that this chainstate //! is based on a snapshot. - CChainState& InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash = std::nullopt) + CChainState& InitializeChainstate( + CTxMemPool* mempool, + const std::optional<uint256>& snapshot_blockhash = std::nullopt) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. diff --git a/test/README.md b/test/README.md index ab34ce42dc..51e61562a4 100644 --- a/test/README.md +++ b/test/README.md @@ -84,6 +84,12 @@ Run all possible tests with test/functional/test_runner.py --extended ``` +In order to run backwards compatibility tests, download the previous node binaries: + +``` +test/get_previous_releases.py -b v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +``` + By default, up to 4 tests will be run in parallel by test_runner. To specify how many jobs to run, append `--jobs=n` diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index fab921ef19..cde0399d8b 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -29,27 +29,32 @@ from test_framework.messages import ( CTxOut, MAX_MONEY, ) -from test_framework import script as sc from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS from test_framework.script import ( CScript, + OP_0, + OP_2DIV, + OP_2MUL, + OP_AND, OP_CAT, - OP_SUBSTR, - OP_LEFT, - OP_RIGHT, + OP_CHECKSIG, + OP_DIV, OP_INVERT, - OP_AND, + OP_LEFT, + OP_LSHIFT, + OP_MOD, + OP_MUL, OP_OR, + OP_RIGHT, + OP_RSHIFT, + OP_SUBSTR, + OP_TRUE, OP_XOR, - OP_2MUL, - OP_2DIV, - OP_MUL, - OP_DIV, - OP_MOD, - OP_LSHIFT, - OP_RSHIFT ) -basic_p2sh = sc.CScript([sc.OP_HASH160, sc.hash160(sc.CScript([sc.OP_0])), sc.OP_EQUAL]) +from test_framework.script_util import ( + script_to_p2sh_script, +) +basic_p2sh = script_to_p2sh_script(CScript([OP_0])) class BadTxTemplate: @@ -116,7 +121,7 @@ class SizeTooSmall(BadTxTemplate): def get_tx(self): tx = CTransaction() tx.vin.append(self.valid_txin) - tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE]))) + tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) tx.calc_sha256() return tx @@ -151,6 +156,19 @@ class DuplicateInput(BadTxTemplate): return tx +class PrevoutNullInput(BadTxTemplate): + reject_reason = 'bad-txns-prevout-null' + expect_disconnect = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vin.append(CTxIn(COutPoint(hash=0, n=0xffffffff))) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + class NonexistentInput(BadTxTemplate): reject_reason = None # Added as an orphan tx. expect_disconnect = False @@ -217,7 +235,7 @@ class TooManySigops(BadTxTemplate): expect_disconnect = False def get_tx(self): - lotsa_checksigs = sc.CScript([sc.OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) + lotsa_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) return create_tx_with_script( self.spend_tx, 0, script_pub_key=lotsa_checksigs, diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index c712f7141c..e0ba835f99 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -4,9 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Backwards compatibility functional test -Test various backwards compatibility scenarios. Download the previous node binaries: - -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +Test various backwards compatibility scenarios. Requires previous releases binaries, +see test/README.md. v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. diff --git a/test/functional/feature_bind_extra.py b/test/functional/feature_bind_extra.py new file mode 100755 index 0000000000..6802da8d48 --- /dev/null +++ b/test/functional/feature_bind_extra.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test starting bitcoind with -bind and/or -bind=...=onion and confirm +that bind happens on the expected ports. +""" + +import sys + +from test_framework.netutil import ( + addr_to_hex, + get_bind_addrs, +) +from test_framework.test_framework import ( + BitcoinTestFramework, + SkipTest, +) +from test_framework.util import ( + PORT_MIN, + PORT_RANGE, + assert_equal, + rpc_port, +) + +class BindExtraTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + # Avoid any -bind= on the command line. Force the framework to avoid + # adding -bind=127.0.0.1. + self.bind_to_localhost_only = False + self.num_nodes = 2 + + def setup_network(self): + # Override setup_network() because we want to put the result of + # p2p_port() in self.extra_args[], before the nodes are started. + # p2p_port() is not usable in set_test_params() because PortSeed.n is + # not set at that time. + + # Due to OS-specific network stats queries, we only run on Linux. + self.log.info("Checking for Linux") + if not sys.platform.startswith('linux'): + raise SkipTest("This test can only be run on Linux.") + + loopback_ipv4 = addr_to_hex("127.0.0.1") + + # Start custom ports after p2p and rpc ports. + port = PORT_MIN + 2 * PORT_RANGE + + # Array of tuples [command line arguments, expected bind addresses]. + self.expected = [] + + # Node0, no normal -bind=... with -bind=...=onion, thus only the tor target. + self.expected.append( + [ + [f"-bind=127.0.0.1:{port}=onion"], + [(loopback_ipv4, port)] + ], + ) + port += 1 + + # Node1, both -bind=... and -bind=...=onion. + self.expected.append( + [ + [f"-bind=127.0.0.1:{port}", f"-bind=127.0.0.1:{port + 1}=onion"], + [(loopback_ipv4, port), (loopback_ipv4, port + 1)] + ], + ) + port += 2 + + self.extra_args = list(map(lambda e: e[0], self.expected)) + self.add_nodes(self.num_nodes, self.extra_args) + # Don't start the nodes, as some of them would collide trying to bind on the same port. + + def run_test(self): + for i in range(len(self.expected)): + self.log.info(f"Starting node {i} with {self.expected[i][0]}") + self.start_node(i) + pid = self.nodes[i].process.pid + binds = set(get_bind_addrs(pid)) + # Remove IPv6 addresses because on some CI environments "::1" is not configured + # on the system (so our test_ipv6_local() would return False), but it is + # possible to bind on "::". This makes it unpredictable whether to expect + # that bitcoind has bound on "::1" (for RPC) and "::" (for P2P). + ipv6_addr_len_bytes = 32 + binds = set(filter(lambda e: len(e[0]) != ipv6_addr_len_bytes, binds)) + # Remove RPC ports. They are not relevant for this test. + binds = set(filter(lambda e: e[1] != rpc_port(i), binds)) + assert_equal(binds, set(self.expected[i][1])) + self.stop_node(i) + self.log.info(f"Stopped node {i}") + +if __name__ == '__main__': + BindExtraTest().main() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 389db73bc9..c11eabc917 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -37,17 +37,17 @@ from test_framework.script import ( OP_CHECKSIGVERIFY, OP_ELSE, OP_ENDIF, - OP_EQUAL, OP_DROP, OP_FALSE, - OP_HASH160, OP_IF, OP_INVALIDOPCODE, OP_RETURN, OP_TRUE, SIGHASH_ALL, LegacySignatureHash, - hash160, +) +from test_framework.script_util import ( + script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -469,8 +469,7 @@ class FullBlockTest(BitcoinTestFramework): # Build the redeem script, hash it, use hash to create the p2sh script redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY] * 5 + [OP_CHECKSIG]) - redeem_script_hash = hash160(redeem_script) - p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL]) + p2sh_script = script_to_p2sh_script(redeem_script) # Create a transaction that spends one satoshi to the p2sh_script, the rest to OP_TRUE # This must be signed because it is spending a coinbase diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 8ccdf87ff3..5322b02414 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -18,10 +18,10 @@ from test_framework.script import ( OP_1, OP_2, OP_DROP, - OP_EQUAL, - OP_HASH160, OP_TRUE, - hash160, +) +from test_framework.script_util import ( + script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -37,8 +37,8 @@ from test_framework.util import ( # time signing. REDEEM_SCRIPT_1 = CScript([OP_1, OP_DROP]) REDEEM_SCRIPT_2 = CScript([OP_2, OP_DROP]) -P2SH_1 = CScript([OP_HASH160, hash160(REDEEM_SCRIPT_1), OP_EQUAL]) -P2SH_2 = CScript([OP_HASH160, hash160(REDEEM_SCRIPT_2), OP_EQUAL]) +P2SH_1 = script_to_p2sh_script(REDEEM_SCRIPT_1) +P2SH_2 = script_to_p2sh_script(REDEEM_SCRIPT_2) # Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 SCRIPT_SIG = [CScript([OP_TRUE, REDEEM_SCRIPT_1]), CScript([OP_TRUE, REDEEM_SCRIPT_2])] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 42910904d7..9cf46d9d11 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -23,7 +23,6 @@ from test_framework.messages import ( CTransaction, CTxIn, CTxOut, - sha256, tx_from_hex, ) from test_framework.script import ( @@ -34,12 +33,13 @@ from test_framework.script import ( OP_CHECKMULTISIG, OP_CHECKSIG, OP_DROP, - OP_DUP, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, OP_TRUE, - hash160, +) +from test_framework.script_util import ( + key_to_p2pkh_script, + key_to_p2wpkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -353,7 +353,7 @@ class SegWitTest(BitcoinTestFramework): multisig_without_privkey_address = self.nodes[0].addmultisigaddress(2, [pubkeys[3], pubkeys[4]])['address'] script = CScript([OP_2, hex_str_to_bytes(pubkeys[3]), hex_str_to_bytes(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) - solvable_after_importaddress.append(CScript([OP_HASH160, hash160(script), OP_EQUAL])) + solvable_after_importaddress.append(script_to_p2sh_script(script)) for i in compressed_spendable_address: v = self.nodes[0].getaddressinfo(i) @@ -427,10 +427,10 @@ class SegWitTest(BitcoinTestFramework): op0 = CScript([OP_0]) # 2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe is the P2SH(P2PKH) version of mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V unsolvable_address_key = hex_str_to_bytes("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") - unsolvablep2pkh = CScript([OP_DUP, OP_HASH160, hash160(unsolvable_address_key), OP_EQUALVERIFY, OP_CHECKSIG]) - unsolvablep2wshp2pkh = CScript([OP_0, sha256(unsolvablep2pkh)]) - p2shop0 = CScript([OP_HASH160, hash160(op0), OP_EQUAL]) - p2wshop1 = CScript([OP_0, sha256(op1)]) + unsolvablep2pkh = key_to_p2pkh_script(unsolvable_address_key) + unsolvablep2wshp2pkh = script_to_p2wsh_script(unsolvablep2pkh) + p2shop0 = script_to_p2sh_script(op0) + p2wshop1 = script_to_p2wsh_script(op1) unsolvable_after_importaddress.append(unsolvablep2pkh) unsolvable_after_importaddress.append(unsolvablep2wshp2pkh) unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script @@ -450,16 +450,16 @@ class SegWitTest(BitcoinTestFramework): if (v['isscript']): bare = hex_str_to_bytes(v['hex']) importlist.append(bare.hex()) - importlist.append(CScript([OP_0, sha256(bare)]).hex()) + importlist.append(script_to_p2wsh_script(bare).hex()) else: pubkey = hex_str_to_bytes(v['pubkey']) p2pk = CScript([pubkey, OP_CHECKSIG]) - p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) + p2pkh = key_to_p2pkh_script(pubkey) importlist.append(p2pk.hex()) importlist.append(p2pkh.hex()) - importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) - importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) - importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) + importlist.append(key_to_p2wpkh_script(pubkey).hex()) + importlist.append(script_to_p2wsh_script(p2pk).hex()) + importlist.append(script_to_p2wsh_script(p2pkh).hex()) importlist.append(unsolvablep2pkh.hex()) importlist.append(unsolvablep2wshp2pkh.hex()) @@ -614,22 +614,22 @@ class SegWitTest(BitcoinTestFramework): def p2sh_address_to_script(self, v): bare = CScript(hex_str_to_bytes(v['hex'])) p2sh = CScript(hex_str_to_bytes(v['scriptPubKey'])) - p2wsh = CScript([OP_0, sha256(bare)]) - p2sh_p2wsh = CScript([OP_HASH160, hash160(p2wsh), OP_EQUAL]) + p2wsh = script_to_p2wsh_script(bare) + p2sh_p2wsh = script_to_p2sh_script(p2wsh) return([bare, p2sh, p2wsh, p2sh_p2wsh]) def p2pkh_address_to_script(self, v): pubkey = hex_str_to_bytes(v['pubkey']) - p2wpkh = CScript([OP_0, hash160(pubkey)]) - p2sh_p2wpkh = CScript([OP_HASH160, hash160(p2wpkh), OP_EQUAL]) + p2wpkh = key_to_p2wpkh_script(pubkey) + p2sh_p2wpkh = script_to_p2sh_script(p2wpkh) p2pk = CScript([pubkey, OP_CHECKSIG]) p2pkh = CScript(hex_str_to_bytes(v['scriptPubKey'])) - p2sh_p2pk = CScript([OP_HASH160, hash160(p2pk), OP_EQUAL]) - p2sh_p2pkh = CScript([OP_HASH160, hash160(p2pkh), OP_EQUAL]) - p2wsh_p2pk = CScript([OP_0, sha256(p2pk)]) - p2wsh_p2pkh = CScript([OP_0, sha256(p2pkh)]) - p2sh_p2wsh_p2pk = CScript([OP_HASH160, hash160(p2wsh_p2pk), OP_EQUAL]) - p2sh_p2wsh_p2pkh = CScript([OP_HASH160, hash160(p2wsh_p2pkh), OP_EQUAL]) + p2sh_p2pk = script_to_p2sh_script(p2pk) + p2sh_p2pkh = script_to_p2sh_script(p2pkh) + p2wsh_p2pk = script_to_p2wsh_script(p2pk) + p2wsh_p2pkh = script_to_p2wsh_script(p2pkh) + p2sh_p2wsh_p2pk = script_to_p2sh_script(p2wsh_p2pk) + p2sh_p2wsh_p2pkh = script_to_p2sh_script(p2wsh_p2pkh) return [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] def create_and_mine_tx_from_txids(self, txids, success=True): diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 99283b69b0..f27ab2057c 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -57,7 +57,6 @@ from test_framework.script import ( OP_ENDIF, OP_EQUAL, OP_EQUALVERIFY, - OP_HASH160, OP_IF, OP_NOP, OP_NOT, @@ -76,12 +75,17 @@ from test_framework.script import ( is_op_success, taproot_construct, ) +from test_framework.script_util import ( + key_to_p2wpkh_script, + keyhash_to_p2pkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, - sha256, ) from collections import OrderedDict, namedtuple from io import BytesIO @@ -458,13 +462,13 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= # P2WPKH assert script is None pubkeyhash = hash160(pkh) - spk = CScript([OP_0, pubkeyhash]) - conf["scriptcode"] = CScript([OP_DUP, OP_HASH160, pubkeyhash, OP_EQUALVERIFY, OP_CHECKSIG]) + spk = key_to_p2wpkh_script(pkh) + conf["scriptcode"] = keyhash_to_p2pkh_script(pubkeyhash) conf["script_witv0"] = None conf["inputs"] = [getter("sign"), pkh] elif script is not None: # P2WSH - spk = CScript([OP_0, sha256(script)]) + spk = script_to_p2wsh_script(script) conf["scriptcode"] = script conf["script_witv0"] = script else: @@ -475,7 +479,7 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= # P2PKH assert script is None pubkeyhash = hash160(pkh) - spk = CScript([OP_DUP, OP_HASH160, pubkeyhash, OP_EQUALVERIFY, OP_CHECKSIG]) + spk = keyhash_to_p2pkh_script(pubkeyhash) conf["scriptcode"] = spk conf["inputs"] = [getter("sign"), pkh] elif script is not None: @@ -496,7 +500,7 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= if p2sh: # P2SH wrapper can be combined with anything else conf["script_p2sh"] = spk - spk = CScript([OP_HASH160, hash160(spk), OP_EQUAL]) + spk = script_to_p2sh_script(spk) conf = {**conf, **kwargs} @@ -518,9 +522,9 @@ def add_spender(spenders, *args, **kwargs): def random_checksig_style(pubkey): """Creates a random CHECKSIG* tapscript that would succeed with only the valid signature on witness stack.""" opcode = random.choice([OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKSIGADD]) - if (opcode == OP_CHECKSIGVERIFY): + if opcode == OP_CHECKSIGVERIFY: ret = CScript([pubkey, opcode, OP_1]) - elif (opcode == OP_CHECKSIGADD): + elif opcode == OP_CHECKSIGADD: num = random.choice([0, 0x7fffffff, -0x7fffffff]) ret = CScript([num, pubkey, opcode, num + 1, OP_EQUAL]) else: @@ -1189,19 +1193,36 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # Data type to keep track of UTXOs, where they were created, and how to spend them. UTXOData = namedtuple('UTXOData', 'outpoint,output,spender') + class TaprootTest(BitcoinTestFramework): def add_options(self, parser): parser.add_argument("--dumptests", dest="dump_tests", default=False, action="store_true", help="Dump generated test cases to directory set by TEST_DUMP_DIR environment variable") + parser.add_argument("--previous_release", dest="previous_release", default=False, action="store_true", + help="Use a previous release as taproot-inactive node") def skip_test_if_missing_module(self): self.skip_if_no_wallet() + if self.options.previous_release: + self.skip_if_no_previous_releases() def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-par=1", "-vbparams=taproot:1:1"], ["-par=1"]] + self.extra_args = [["-par=1"], ["-par=1"]] + if self.options.previous_release: + self.wallet_names = [None, self.default_wallet_name] + else: + self.extra_args[0].append("-vbparams=taproot:1:1") + + def setup_nodes(self): + self.add_nodes(self.num_nodes, self.extra_args, versions=[ + 200100 if self.options.previous_release else None, + None, + ]) + self.start_nodes() + self.import_deterministic_coinbase_privkeys() def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1223,7 +1244,7 @@ class TaprootTest(BitcoinTestFramework): block_response = node.submitblock(block.serialize().hex()) if err_msg is not None: assert block_response is not None and err_msg in block_response, "Missing error message '%s' from block response '%s': %s" % (err_msg, "(None)" if block_response is None else block_response, msg) - if (accept): + if accept: assert node.getbestblockhash() == block.hash, "Failed to accept: %s (response: %s)" % (msg, block_response) self.tip = block.sha256 self.lastblockhash = block.hash diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 1705d957aa..60c0953f6f 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -13,22 +13,24 @@ from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, COutPoint, + CTxIn, CTxOut, MAX_BLOCK_BASE_SIZE, MAX_MONEY, tx_from_hex, ) from test_framework.script import ( - hash160, CScript, OP_0, OP_2, OP_3, OP_CHECKMULTISIG, - OP_EQUAL, OP_HASH160, OP_RETURN, ) +from test_framework.script_util import ( + script_to_p2sh_script, +) from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -247,6 +249,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) + self.log.info('A non-coinbase transaction with coinbase-like outpoint') + tx = tx_from_hex(raw_tx_reference) + tx.vin.append(CTxIn(COutPoint(hash=0, n=0xffffffff))) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-prevout-null'}], + rawtxs=[tx.serialize().hex()], + ) + self.log.info('A coinbase transaction') # Pick the input of the first tx we signed, so it has to be a coinbase tx raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid']) @@ -291,7 +301,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx = tx_from_hex(raw_tx_reference) - output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) + output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=script_to_p2sh_script(b'burn')) num_scripts = 100000 // len(output_p2sh_burn.serialize()) # Use enough outputs to make the tx too large for our policy tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( diff --git a/test/functional/mempool_accept_wtxid.py b/test/functional/mempool_accept_wtxid.py new file mode 100755 index 0000000000..dd1f8997ad --- /dev/null +++ b/test/functional/mempool_accept_wtxid.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test mempool acceptance in case of an already known transaction +with identical non-witness data different witness. +""" + +from test_framework.messages import ( + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxInWitness, + CTxOut, + sha256, +) +from test_framework.script import ( + CScript, + OP_0, + OP_ELSE, + OP_ENDIF, + OP_EQUAL, + OP_HASH160, + OP_IF, + OP_TRUE, + hash160, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class MempoolWtxidTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + node = self.nodes[0] + + self.log.info('Start with empty mempool and 101 blocks') + # The last 100 coinbase transactions are premature + blockhash = node.generate(101)[0] + txid = node.getblock(blockhash=blockhash, verbosity=2)["tx"][0]["txid"] + assert_equal(node.getmempoolinfo()['size'], 0) + + self.log.info("Submit parent with multiple script branches to mempool") + hashlock = hash160(b'Preimage') + witness_script = CScript([OP_IF, OP_HASH160, hashlock, OP_EQUAL, OP_ELSE, OP_TRUE, OP_ENDIF]) + witness_program = sha256(witness_script) + script_pubkey = CScript([OP_0, witness_program]) + + parent = CTransaction() + parent.vin.append(CTxIn(COutPoint(int(txid, 16), 0), b"")) + parent.vout.append(CTxOut(int(9.99998 * COIN), script_pubkey)) + parent.rehash() + + privkeys = [node.get_deterministic_priv_key().key] + raw_parent = node.signrawtransactionwithkey(hexstring=parent.serialize().hex(), privkeys=privkeys)['hex'] + parent_txid = node.sendrawtransaction(hexstring=raw_parent, maxfeerate=0) + node.generate(1) + + # Create a new transaction with witness solving first branch + child_witness_script = CScript([OP_TRUE]) + child_witness_program = sha256(child_witness_script) + child_script_pubkey = CScript([OP_0, child_witness_program]) + + child_one = CTransaction() + child_one.vin.append(CTxIn(COutPoint(int(parent_txid, 16), 0), b"")) + child_one.vout.append(CTxOut(int(9.99996 * COIN), child_script_pubkey)) + child_one.wit.vtxinwit.append(CTxInWitness()) + child_one.wit.vtxinwit[0].scriptWitness.stack = [b'Preimage', b'\x01', witness_script] + child_one_wtxid = child_one.getwtxid() + child_one_txid = child_one.rehash() + + # Create another identical transaction with witness solving second branch + child_two = CTransaction() + child_two.vin.append(CTxIn(COutPoint(int(parent_txid, 16), 0), b"")) + child_two.vout.append(CTxOut(int(9.99996 * COIN), child_script_pubkey)) + child_two.wit.vtxinwit.append(CTxInWitness()) + child_two.wit.vtxinwit[0].scriptWitness.stack = [b'', witness_script] + child_two_wtxid = child_two.getwtxid() + child_two_txid = child_two.rehash() + + assert_equal(child_one_txid, child_two_txid) + assert child_one_wtxid != child_two_wtxid + + self.log.info("Submit one child to the mempool") + txid_submitted = node.sendrawtransaction(child_one.serialize().hex()) + assert_equal(node.getrawmempool(True)[txid_submitted]['wtxid'], child_one_wtxid) + + # testmempoolaccept reports the "already in mempool" error + assert_equal(node.testmempoolaccept([child_one.serialize().hex()]), [{ + "txid": child_one_txid, + "wtxid": child_one_wtxid, + "allowed": False, + "reject-reason": "txn-already-in-mempool" + }]) + testres_child_two = node.testmempoolaccept([child_two.serialize().hex()])[0] + assert_equal(testres_child_two, { + "txid": child_two_txid, + "wtxid": child_two_wtxid, + "allowed": False, + "reject-reason": "txn-same-nonwitness-data-in-mempool" + }) + + # sendrawtransaction will not throw but quits early when the exact same transaction is already in mempool + node.sendrawtransaction(child_one.serialize().hex()) + # sendrawtransaction will not throw but quits early when a transaction with the same non-witness data is already in mempool + node.sendrawtransaction(child_two.serialize().hex()) + + +if __name__ == '__main__': + MempoolWtxidTest().main() diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 6de6778909..87f40b7f2b 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -7,10 +7,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken accidentally. In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. -Download node binaries: -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 - -Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. +The previous release v0.15.2 is required by this test, see test/README.md. """ import os diff --git a/test/functional/p2p_i2p_ports.py b/test/functional/p2p_i2p_ports.py new file mode 100755 index 0000000000..13188b9305 --- /dev/null +++ b/test/functional/p2p_i2p_ports.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test ports handling for I2P hosts +""" + +import re + +from test_framework.test_framework import BitcoinTestFramework + + +class I2PPorts(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + # The test assumes that an I2P SAM proxy is not listening here. + self.extra_args = [["-i2psam=127.0.0.1:60000"]] + + def run_test(self): + node = self.nodes[0] + + self.log.info("Ensure we don't try to connect if port!=0") + addr = "zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:8333" + raised = False + try: + with node.assert_debug_log(expected_msgs=[f"Error connecting to {addr}"]): + node.addnode(node=addr, command="onetry") + except AssertionError as e: + raised = True + if not re.search(r"Expected messages .* does not partially match log", str(e)): + raise AssertionError(f"Assertion raised as expected, but with an unexpected message: {str(e)}") + if not raised: + raise AssertionError("Assertion should have been raised") + + self.log.info("Ensure we try to connect if port=0 and get an error due to missing I2P proxy") + addr = "h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0" + with node.assert_debug_log(expected_msgs=[f"Error connecting to {addr}"]): + node.addnode(node=addr, command="onetry") + + +if __name__ == '__main__': + I2PPorts().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 95c7aec318..ead9d852fe 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -41,7 +41,6 @@ from test_framework.messages import ( ser_vector, sha256, tx_from_hex, - uint256_from_str, ) from test_framework.p2p import ( P2PInterface, @@ -60,12 +59,8 @@ from test_framework.script import ( OP_CHECKMULTISIG, OP_CHECKSIG, OP_DROP, - OP_DUP, OP_ELSE, OP_ENDIF, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, OP_IF, OP_RETURN, OP_TRUE, @@ -77,6 +72,12 @@ from test_framework.script import ( LegacySignatureHash, hash160, ) +from test_framework.script_util import ( + key_to_p2wpkh_script, + keyhash_to_p2pkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -100,10 +101,6 @@ class UTXO(): self.n = n self.nValue = value -def get_p2pkh_script(pubkeyhash): - """Get the script associated with a P2PKH.""" - return CScript([CScriptOp(OP_DUP), CScriptOp(OP_HASH160), pubkeyhash, CScriptOp(OP_EQUALVERIFY), CScriptOp(OP_CHECKSIG)]) - def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): """Add signature for a P2PK witness program.""" tx_hash = SegwitV0SignatureHash(script, tx_to, in_idx, hashtype, value) @@ -492,11 +489,8 @@ class SegWitTest(BitcoinTestFramework): # Create two outputs, a p2wsh and p2sh-p2wsh witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) - - p2sh_pubkey = hash160(script_pubkey) - p2sh_script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + script_pubkey = script_to_p2wsh_script(witness_program) + p2sh_script_pubkey = script_to_p2sh_script(script_pubkey) value = self.utxo[0].nValue // 3 @@ -631,11 +625,8 @@ class SegWitTest(BitcoinTestFramework): V0 segwit inputs may only be mined after activation, but not before.""" witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) - - p2sh_pubkey = hash160(witness_program) - p2sh_script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + script_pubkey = script_to_p2wsh_script(witness_program) + p2sh_script_pubkey = script_to_p2sh_script(witness_program) # First prepare a p2sh output (so that spending it will pass standardness) p2sh_tx = CTransaction() @@ -662,6 +653,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[1], self.std_node, tx, with_witness=True, accepted=True) # Now create something that looks like a P2PKH output. This won't be spendable. + witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, hash160(witness_hash)]) tx2 = CTransaction() # tx was accepted, so we spend the second output. @@ -740,10 +732,8 @@ class SegWitTest(BitcoinTestFramework): # Prepare the p2sh-wrapped witness output witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - p2wsh_pubkey = CScript([OP_0, witness_hash]) - p2sh_witness_hash = hash160(p2wsh_pubkey) - script_pubkey = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + p2wsh_pubkey = script_to_p2wsh_script(witness_program) + script_pubkey = script_to_p2sh_script(p2wsh_pubkey) script_sig = CScript([p2wsh_pubkey]) # a push of the redeem script # Fund the P2SH output @@ -837,8 +827,7 @@ class SegWitTest(BitcoinTestFramework): # Let's construct a witness program witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) tx.vout.append(CTxOut(self.utxo[0].nValue - 1000, script_pubkey)) tx.rehash() @@ -951,8 +940,7 @@ class SegWitTest(BitcoinTestFramework): NUM_OUTPUTS = 50 witness_program = CScript([OP_2DROP] * NUM_DROPS + [OP_TRUE]) - witness_hash = uint256_from_str(sha256(witness_program)) - script_pubkey = CScript([OP_0, ser_uint256(witness_hash)]) + script_pubkey = script_to_p2wsh_script(witness_program) prevout = COutPoint(self.utxo[0].sha256, self.utxo[0].n) value = self.utxo[0].nValue @@ -1054,8 +1042,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) # First try extra witness data on a tx that doesn't require a witness tx = CTransaction() @@ -1127,8 +1114,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) @@ -1166,8 +1152,7 @@ class SegWitTest(BitcoinTestFramework): # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. long_witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 63 + [OP_TRUE]) assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 - long_witness_hash = sha256(long_witness_program) - long_script_pubkey = CScript([OP_0, long_witness_hash]) + long_script_pubkey = script_to_p2wsh_script(long_witness_program) block = self.build_next_block() @@ -1190,8 +1175,7 @@ class SegWitTest(BitcoinTestFramework): # Try again with one less byte in the witness program witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) assert len(witness_program) == MAX_PROGRAM_LENGTH - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) tx.vout[0] = CTxOut(tx.vout[0].nValue, script_pubkey) tx.rehash() @@ -1210,8 +1194,7 @@ class SegWitTest(BitcoinTestFramework): """Test that vin length must match vtxinwit length.""" witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) # Create a transaction that splits our utxo into many outputs tx = CTransaction() @@ -1318,8 +1301,7 @@ class SegWitTest(BitcoinTestFramework): # Now try to add extra witness data to a valid witness tx. witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx_hash, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_pubkey)) @@ -1331,9 +1313,8 @@ class SegWitTest(BitcoinTestFramework): # Add too-large for IsStandard witness and check that it does not enter reject filter p2sh_program = CScript([OP_TRUE]) - p2sh_pubkey = hash160(p2sh_program) witness_program2 = CScript([b'a' * 400000]) - tx3.vout.append(CTxOut(tx2.vout[0].nValue - 1000, CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]))) + tx3.vout.append(CTxOut(tx2.vout[0].nValue - 1000, script_to_p2sh_script(p2sh_program))) tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2] tx3.rehash() @@ -1482,8 +1463,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() # Change the output of the block to be a witness output. witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) block.vtx[0].vout[0].scriptPubKey = script_pubkey # This next line will rehash the coinbase and update the merkle # root, and solve. @@ -1530,7 +1510,7 @@ class SegWitTest(BitcoinTestFramework): # Test 1: P2WPKH # First create a P2WPKH output that uses an uncompressed pubkey pubkeyhash = hash160(pubkey) - script_pkh = CScript([OP_0, pubkeyhash]) + script_pkh = key_to_p2wpkh_script(pubkey) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(utxo.sha256, utxo.n), b"")) tx.vout.append(CTxOut(utxo.nValue - 1000, script_pkh)) @@ -1544,13 +1524,12 @@ class SegWitTest(BitcoinTestFramework): # Now try to spend it. Send it to a P2WSH output, which we'll # use in the next test. witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - script_wsh = CScript([OP_0, witness_hash]) + script_wsh = script_to_p2wsh_script(witness_program) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_wsh)) - script = get_p2pkh_script(pubkeyhash) + script = keyhash_to_p2pkh_script(pubkeyhash) sig_hash = SegwitV0SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx2.wit.vtxinwit.append(CTxInWitness()) @@ -1567,8 +1546,7 @@ class SegWitTest(BitcoinTestFramework): # Test 2: P2WSH # Try to spend the P2WSH output created in last test. # Send it to a P2SH(P2WSH) output, which we'll use in the next test. - p2sh_witness_hash = hash160(script_wsh) - script_p2sh = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + script_p2sh = script_to_p2sh_script(script_wsh) script_sig = CScript([script_wsh]) tx3 = CTransaction() @@ -1587,7 +1565,7 @@ class SegWitTest(BitcoinTestFramework): # Test 3: P2SH(P2WSH) # Try to spend the P2SH output created in the last test. # Send it to a P2PKH output, which we'll use in the next test. - script_pubkey = get_p2pkh_script(pubkeyhash) + script_pubkey = keyhash_to_p2pkh_script(pubkeyhash) tx4 = CTransaction() tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), script_sig)) tx4.vout.append(CTxOut(tx3.vout[0].nValue - 1000, script_pubkey)) @@ -1624,8 +1602,7 @@ class SegWitTest(BitcoinTestFramework): pubkey = key.get_pubkey().get_bytes() witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) # First create a witness output for use in the tests. tx = CTransaction() @@ -1744,7 +1721,7 @@ class SegWitTest(BitcoinTestFramework): # Now test witness version 0 P2PKH transactions pubkeyhash = hash160(pubkey) - script_pkh = CScript([OP_0, pubkeyhash]) + script_pkh = key_to_p2wpkh_script(pubkey) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(temp_utxos[0].sha256, temp_utxos[0].n), b"")) tx.vout.append(CTxOut(temp_utxos[0].nValue, script_pkh)) @@ -1754,7 +1731,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) - script = get_p2pkh_script(pubkeyhash) + script = keyhash_to_p2pkh_script(pubkeyhash) sig_hash = SegwitV0SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL @@ -1806,8 +1783,7 @@ class SegWitTest(BitcoinTestFramework): # rules (an anyone-can-spend OP_TRUE would be rejected, if not wrapped # in P2SH). p2sh_program = CScript([OP_TRUE]) - p2sh_pubkey = hash160(p2sh_program) - script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + script_pubkey = script_to_p2sh_script(p2sh_program) # Now check that unnecessary witnesses can't be used to blind a node # to a transaction, eg by violating standardness checks. @@ -1872,11 +1848,10 @@ class SegWitTest(BitcoinTestFramework): # For each script, generate a pair of P2WSH and P2SH-P2WSH output. outputvalue = (self.utxo[0].nValue - 1000) // (len(scripts) * 2) for i in scripts: - p2wsh = CScript([OP_0, sha256(i)]) - p2sh = hash160(p2wsh) + p2wsh = script_to_p2wsh_script(i) p2wsh_scripts.append(p2wsh) tx.vout.append(CTxOut(outputvalue, p2wsh)) - tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) + tx.vout.append(CTxOut(outputvalue, script_to_p2sh_script(p2wsh))) tx.rehash() txid = tx.sha256 test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) @@ -1890,13 +1865,13 @@ class SegWitTest(BitcoinTestFramework): for i in range(len(scripts)): p2wsh_tx = CTransaction() p2wsh_tx.vin.append(CTxIn(COutPoint(txid, i * 2))) - p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(b"")]))) p2wsh_tx.wit.vtxinwit.append(CTxInWitness()) p2wsh_tx.rehash() p2wsh_txs.append(p2wsh_tx) p2sh_tx = CTransaction() p2sh_tx.vin.append(CTxIn(COutPoint(txid, i * 2 + 1), CScript([p2wsh_scripts[i]]))) - p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(b"")]))) p2sh_tx.wit.vtxinwit.append(CTxInWitness()) p2sh_tx.rehash() p2sh_txs.append(p2sh_tx) @@ -1991,8 +1966,7 @@ class SegWitTest(BitcoinTestFramework): # Keep this under MAX_OPS_PER_SCRIPT (201) witness_program = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKMULTISIG] * 5 + [OP_CHECKSIG] * 193 + [OP_ENDIF]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) sigops_per_script = 20 * 5 + 193 * 1 # We'll produce 2 extra outputs, one with a program that would take us @@ -2008,14 +1982,12 @@ class SegWitTest(BitcoinTestFramework): # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, # would push us just over the block sigop limit. witness_program_toomany = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available + 1) + [OP_ENDIF]) - witness_hash_toomany = sha256(witness_program_toomany) - script_pubkey_toomany = CScript([OP_0, witness_hash_toomany]) + script_pubkey_toomany = script_to_p2wsh_script(witness_program_toomany) # If we spend this script instead, we would exactly reach our sigop # limit (for witness sigops). witness_program_justright = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available) + [OP_ENDIF]) - witness_hash_justright = sha256(witness_program_justright) - script_pubkey_justright = CScript([OP_0, witness_hash_justright]) + script_pubkey_justright = script_to_p2wsh_script(witness_program_justright) # First split our available utxo into a bunch of outputs split_value = self.utxo[0].nValue // outputs @@ -2148,8 +2120,7 @@ class SegWitTest(BitcoinTestFramework): # Create a Segwit output from the latest UTXO # and announce it to the network witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + script_pubkey = script_to_p2wsh_script(witness_program) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index db57368eae..3ff74dc5a4 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -56,6 +56,10 @@ class RawTransactionsTest(BitcoinTestFramework): ["-txindex"], ["-txindex"], ] + # whitelist all peers to speed up tx relay / mempool sync + for args in self.extra_args: + args.append("-whitelist=noban@127.0.0.1") + self.supports_cli = False def skip_test_if_missing_module(self): diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index ef5d08e7b9..f3627d1e37 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -6,7 +6,6 @@ from test_framework.blocktools import COINBASE_MATURITY from test_framework.address import ( - check_script, script_to_p2sh, script_to_p2wsh, ) @@ -20,12 +19,10 @@ from test_framework.util import ( ) from test_framework.messages import ( CTxInWitness, - sha256, tx_from_hex, ) from test_framework.script import ( CScript, - OP_0, OP_CHECKLOCKTIMEVERIFY, OP_CHECKSIG, OP_CHECKSEQUENCEVERIFY, @@ -233,7 +230,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), 'P2PK': CScript([hex_str_to_bytes(embedded_pubkey), OP_CHECKSIG]).hex() }.get(tx_type, "Invalid tx_type") - redeem_script = CScript([OP_0, sha256(check_script(witness_script))]).hex() + redeem_script = script_to_p2wsh_script(witness_script).hex() addr = script_to_p2sh(redeem_script) script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] # Fund that address diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index bc43438810..833a215993 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -26,7 +26,6 @@ from .messages import ( hash256, hex_str_to_bytes, ser_uint256, - sha256, tx_from_hex, uint256_from_str, ) @@ -34,13 +33,15 @@ from .script import ( CScript, CScriptNum, CScriptOp, - OP_0, OP_1, OP_CHECKMULTISIG, OP_CHECKSIG, OP_RETURN, OP_TRUE, - hash160, +) +from .script_util import ( + key_to_p2wpkh_script, + script_to_p2wsh_script, ) from .util import assert_equal @@ -206,13 +207,11 @@ def witness_script(use_p2wsh, pubkey): scriptPubKey.""" if not use_p2wsh: # P2WPKH instead - pubkeyhash = hash160(hex_str_to_bytes(pubkey)) - pkscript = CScript([OP_0, pubkeyhash]) + pkscript = key_to_p2wpkh_script(pubkey) else: # 1-of-1 multisig witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) - scripthash = sha256(witness_program) - pkscript = CScript([OP_0, scripthash]) + pkscript = script_to_p2wsh_script(witness_program) return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index b9c0fb6691..acbc040741 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -17,17 +17,15 @@ from test_framework.address import ( from test_framework.key import ECKey from test_framework.script import ( CScript, - OP_0, OP_2, OP_3, OP_CHECKMULTISIG, - OP_CHECKSIG, - OP_DUP, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, - hash160, - sha256, +) +from test_framework.script_util import ( + key_to_p2pkh_script, + key_to_p2wpkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, ) from test_framework.util import hex_str_to_bytes @@ -57,15 +55,14 @@ def get_key(node): Returns a named tuple of privkey, pubkey and all address and scripts.""" addr = node.getnewaddress() pubkey = node.getaddressinfo(addr)['pubkey'] - pkh = hash160(hex_str_to_bytes(pubkey)) return Key(privkey=node.dumpprivkey(addr), pubkey=pubkey, - p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_script=key_to_p2pkh_script(pubkey).hex(), p2pkh_addr=key_to_p2pkh(pubkey), - p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_script=key_to_p2wpkh_script(pubkey).hex(), p2wpkh_addr=key_to_p2wpkh(pubkey), - p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), - p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_script=script_to_p2sh_script(key_to_p2wpkh_script(pubkey)).hex(), + p2sh_p2wpkh_redeem_script=key_to_p2wpkh_script(pubkey).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) def get_generate_key(): @@ -76,15 +73,14 @@ def get_generate_key(): eckey.generate() privkey = bytes_to_wif(eckey.get_bytes()) pubkey = eckey.get_pubkey().get_bytes().hex() - pkh = hash160(hex_str_to_bytes(pubkey)) return Key(privkey=privkey, pubkey=pubkey, - p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_script=key_to_p2pkh_script(pubkey).hex(), p2pkh_addr=key_to_p2pkh(pubkey), - p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_script=key_to_p2wpkh_script(pubkey).hex(), p2wpkh_addr=key_to_p2wpkh(pubkey), - p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), - p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_script=script_to_p2sh_script(key_to_p2wpkh_script(pubkey)).hex(), + p2sh_p2wpkh_redeem_script=key_to_p2wpkh_script(pubkey).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) def get_multisig(node): @@ -98,15 +94,15 @@ def get_multisig(node): addrs.append(addr['address']) pubkeys.append(addr['pubkey']) script_code = CScript([OP_2] + [hex_str_to_bytes(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) - witness_script = CScript([OP_0, sha256(script_code)]) + witness_script = script_to_p2wsh_script(script_code) return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs], pubkeys=pubkeys, - p2sh_script=CScript([OP_HASH160, hash160(script_code), OP_EQUAL]).hex(), + p2sh_script=script_to_p2sh_script(script_code).hex(), p2sh_addr=script_to_p2sh(script_code), redeem_script=script_code.hex(), p2wsh_script=witness_script.hex(), p2wsh_addr=script_to_p2wsh(script_code), - p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(), + p2sh_p2wsh_script=script_to_p2sh_script(witness_script).hex(), p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code)) def test_address(node, address, **kwargs): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index ad1acd2e2f..8afd8b3bc1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -111,6 +111,7 @@ BASE_SCRIPTS = [ 'wallet_dump.py --legacy-wallet', 'wallet_listtransactions.py --legacy-wallet', 'wallet_listtransactions.py --descriptors', + 'feature_taproot.py --previous_release', 'feature_taproot.py', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -124,8 +125,6 @@ BASE_SCRIPTS = [ 'wallet_abandonconflict.py --legacy-wallet', 'wallet_abandonconflict.py --descriptors', 'feature_csv_activation.py', - 'rpc_rawtransaction.py --legacy-wallet', - 'rpc_rawtransaction.py --descriptors', 'wallet_address_types.py --legacy-wallet', 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', @@ -139,6 +138,7 @@ BASE_SCRIPTS = [ 'interface_zmq.py', 'rpc_invalid_address_message.py', 'interface_bitcoin_cli.py', + 'feature_bind_extra.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', @@ -171,6 +171,8 @@ BASE_SCRIPTS = [ 'feature_proxy.py', 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', + 'rpc_rawtransaction.py --legacy-wallet', + 'rpc_rawtransaction.py --descriptors', 'wallet_groups.py --legacy-wallet', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', @@ -279,6 +281,7 @@ BASE_SCRIPTS = [ 'feature_asmap.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', + 'mempool_accept_wtxid.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', @@ -291,6 +294,7 @@ BASE_SCRIPTS = [ 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', + 'p2p_i2p_ports.py', 'feature_config_args.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index bf53c99855..c2565d84f6 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -30,9 +30,10 @@ class ListDescriptorsTest(BitcoinTestFramework): node = self.nodes[0] assert_raises_rpc_error(-18, 'No wallet is loaded.', node.listdescriptors) - self.log.info('Test that the command is not available for legacy wallets.') - node.createwallet(wallet_name='w1', descriptors=False) - assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) + if self.is_bdb_compiled(): + self.log.info('Test that the command is not available for legacy wallets.') + node.createwallet(wallet_name='w1', descriptors=False) + assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) self.log.info('Test the command for empty descriptors wallet.') node.createwallet(wallet_name='w2', blank=True, descriptors=True) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index e7dd7592b6..ad11f4b244 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,9 +6,8 @@ Test upgradewallet RPC. Download node binaries: -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 - -Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py +Requires previous releases binaries, see test/README.md. +Only v0.15.2 and v0.16.3 are required by this test. """ import os diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index b177fbb4b2..01e4ef47a7 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -21,39 +21,47 @@ import hashlib SHA256_SUMS = { -"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", -"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", -"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", -"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", -"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", - -"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", -"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", -"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", -"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", -"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", - -"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", -"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", -"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", -"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", -"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", - -"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", -"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", -"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", -"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", -"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", -"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", - -"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", -"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", -"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", -"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", -"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", -"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz" + "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", + "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", + "2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", + "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", + "566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", + # + "0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", + "fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", + "75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", + "78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", + "5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", + # + "5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", + "d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", + "d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", + "a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", + "943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", + # + "88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", + "cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", + "989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", + "b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", + "425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", + "600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", + # + "3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", + "657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", + "10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", + "1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", + "aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", + "5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz", + # + "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz", + "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz", + "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": "bitcoin-0.20.1-osx64.tar.gz", + "c378d4e21109f09e8829f3591e015c66632dff2925a60b64d259be05a334c30b": "bitcoin-0.20.1-osx.dmg", + "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz", + "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz", } + @contextlib.contextmanager def pushd(new_dir) -> None: previous_dir = os.getcwd() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 51815963f6..c448fa6f9a 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -102,7 +102,7 @@ if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; e EXIT_CODE=1 fi -if ! mypy --ignore-missing-imports $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then +if ! mypy --ignore-missing-imports --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then EXIT_CODE=1 fi diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index fbdf3c59c1..238fa63c45 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/gitian-keys/keys.txt"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/gitian-keys/keys.txt" ":(exclude)contrib/guix/patches"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi |