diff options
68 files changed, 1364 insertions, 336 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5c322fc23..0ea479b667 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,12 +153,10 @@ jobs: - name: Get tool information run: | - msbuild -version | Out-File -FilePath "$env:GITHUB_WORKSPACE\msbuild_version" - Get-Content -Path "$env:GITHUB_WORKSPACE\msbuild_version" - $env:VCToolsVersion | Out-File -FilePath "$env:GITHUB_WORKSPACE\toolset_version" - Write-Host "VCToolsVersion $(Get-Content -Path "$env:GITHUB_WORKSPACE\toolset_version")" - $env:CI_QT_URL | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_url" - $env:CI_QT_CONF | Out-File -FilePath "$env:GITHUB_WORKSPACE\qt_conf" + msbuild -version | Tee-Object -FilePath "msbuild_version" + $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" + $env:CI_QT_URL | Out-File -FilePath "qt_url" + $env:CI_QT_CONF | Out-File -FilePath "qt_conf" py -3 --version Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" @@ -241,10 +239,8 @@ jobs: run: | Set-Location "$env:VCPKG_INSTALLATION_ROOT" Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" - Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $env:VCToolsVersion)" .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install - git rev-parse HEAD | Out-File -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - Get-Content -Path "$env:GITHUB_WORKSPACE\vcpkg_commit" + git rev-parse HEAD | Tee-Object -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit" - name: vcpkg tools cache uses: actions/cache@v4 diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index 31c4bff6ae..f607c93ae6 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export CONTAINER_NAME=ci_macos_cross -export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" +export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export HOST=x86_64-apple-darwin -export PACKAGES="zip" +export PACKAGES="clang lld llvm zip" export XCODE_VERSION=15.0 export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false diff --git a/configure.ac b/configure.ac index a1d9e418f8..ab369cc98a 100644 --- a/configure.ac +++ b/configure.ac @@ -206,9 +206,9 @@ AC_ARG_WITH([qrencode], AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening], - [do not attempt to harden the resulting executables (default is to harden when possible)])], + [do not attempt to harden the resulting executables (default is to harden)])], [use_hardening=$enableval], - [use_hardening=auto]) + [use_hardening=yes]) AC_ARG_ENABLE([reduce-exports], [AS_HELP_STRING([--enable-reduce-exports], @@ -281,13 +281,6 @@ AC_ARG_WITH([sanitizers], [comma separated list of extra sanitizers to build with (default is none enabled)])], [use_sanitizers=$withval]) -dnl Enable gprof profiling -AC_ARG_ENABLE([gprof], - [AS_HELP_STRING([--enable-gprof], - [use gprof profiling compiler flags (default is no)])], - [enable_gprof=$enableval], - [enable_gprof=no]) - dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], @@ -411,12 +404,12 @@ AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsug AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wself-assign], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wself-assign"], [], [$CXXFLAG_WERROR]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) @@ -844,30 +837,12 @@ if test "$ac_cv_sys_large_files" != "" && CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test "$enable_gprof" = "yes"; then - dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, - dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force - dnl -pie by default, in which case it needs to be turned off with -no-pie. - - if test "$use_hardening" = "yes"; then - AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) - fi - use_hardening=no - AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - - AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) -fi - if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi if test "$use_hardening" != "no"; then - use_hardening=yes AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) @@ -1684,8 +1659,6 @@ AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) AC_SUBST(ERROR_CXXFLAGS) -AC_SUBST(GPROF_CXXFLAGS) -AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) @@ -1793,7 +1766,6 @@ echo " with natpmp = $use_natpmp" echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" -echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" echo echo " target os = $host_os" @@ -1803,8 +1775,8 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $SANITIZER_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $CORE_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $SANITIZER_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $SANITIZER_LDFLAGS $CORE_LDFLAGS $LDFLAGS" echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index e4a62c2072..6613874ce3 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -239,8 +239,8 @@ def check_MACHO_sdk(binary) -> bool: return True return False -def check_MACHO_ld64(binary) -> bool: - if binary.build_version.tools[0].version == [17, 0, 6]: +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [18, 1, 6]: return True return False @@ -282,7 +282,7 @@ lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), - ('LD64', check_MACHO_ld64), + ('LLD', check_MACHO_lld), ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index f589ac7a55..9bc8c0e75d 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -178,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - FORCE_USE_SYSTEM_CLANG=1 + x86_64_linux_STRIP=x86_64-linux-gnu-strip ########################### diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index ce6a9562b4..80bfb2875f 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ + --commit=f0bb724211872cd6158fce6162e0b8c73efed126 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 40500ccb88..53569d7f7d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -532,9 +532,9 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "darwin") (list ;; Native GCC 11 toolchain gcc-toolchain-11 - clang-toolchain-17 - lld-17 - (make-lld-wrapper lld-17 #:lld-as-ld? #t) + clang-toolchain-18 + lld-18 + (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/depends/Makefile b/depends/Makefile index fee426d8db..52a9a14e56 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -181,8 +181,6 @@ all_packages = $(packages) $(native_packages) meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk -$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) - include funcs.mk final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) diff --git a/depends/README.md b/depends/README.md index 7ac075f4ba..adaeed468f 100644 --- a/depends/README.md +++ b/depends/README.md @@ -49,11 +49,11 @@ The paths are automatically configured and no other options are needed. #### For macOS cross compilation - sudo apt-get install g++ zip + apt install clang lld llvm g++ zip -Note: You must obtain the macOS SDK before proceeding with a cross-compile. -Under the depends directory, create a subdirectory named `SDKs`. -Then, place the extracted SDK under this new directory. +Clang 18 or later is required. You must also obtain the macOS SDK before +proceeding with a cross-compile. Under the depends directory, create a +subdirectory named `SDKs`. Then, place the extracted SDK under this new directory. For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction). #### For Win64 cross compilation @@ -119,9 +119,6 @@ The following can be set when running make: `make FOO=bar` - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids -- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the - system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required - `LOG`: Use file-based logging for individual packages. During a package build its log file resides in the `depends` directory, and the log file is printed out automatically in case of build error. After successful build log files are moved along with package archives diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index d84c23ed44..2b59353e84 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip) darwin_OBJDUMP:=$(shell xcrun -f objdump) darwin_NM:=$(shell xcrun -f nm) darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) -darwin_native_toolchain= x86_64_darwin_CFLAGS += -arch x86_64 x86_64_darwin_CXXFLAGS += -arch x86_64 diff --git a/depends/funcs.mk b/depends/funcs.mk index 537051c030..3c0dc7a7fc 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -46,7 +46,7 @@ endef define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) -$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies))) +$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) @@ -297,6 +297,3 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ #create build targets $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) - -#special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a64008d6aa..f2e9abdf37 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -2,39 +2,10 @@ OSX_MIN_VERSION=11.0 OSX_SDK_VERSION=14.0 XCODE_VERSION=15.0 XCODE_BUILD_ID=15A240d -LD64_VERSION=711 +LLD_VERSION=711 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned LLVM -# from llvm.org - -darwin_native_toolchain=native_llvm - -clang_prog=$(build_prefix)/bin/clang -clangxx_prog=$(clang_prog)++ -llvm_config_prog=$(build_prefix)/bin/llvm-config - -llvm_TOOLS=AR NM OBJDUMP RANLIB STRIP - -# Make-only lowercase function -lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) - -# For well-known tools provided by LLVM, make sure that their well-known -# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} -# would. -$(foreach TOOL,$(llvm_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/llvm-$(call lc,$(TOOL)))) - -# Clang expects dsymutil to be called dsymutil -darwin_DSYMUTIL=$(build_prefix)/bin/dsymutil - -else -# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's -# system - -darwin_native_toolchain= - # We can't just use $(shell command -v clang) because GNU Make handles builtins # in a special way and doesn't know that `command` is a POSIX-standard builtin # prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. @@ -44,9 +15,6 @@ darwin_native_toolchain= # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") -llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") - -llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar") darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil") @@ -54,7 +22,6 @@ darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm") darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump") darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib") darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip") -endif # Flag explanations: # @@ -63,11 +30,6 @@ endif # Ensures that modern linker features are enabled. See here for more # details: https://github.com/bitcoin/bitcoin/pull/19407. # -# -B$(build_prefix)/bin -# -# Explicitly point to our binaries so that they are -# ensured to be found and preferred over other possibilities. -# # -isysroot$(OSX_SDK) -nostdlibinc # # Disable default include paths built into the compiler as well as @@ -92,7 +54,6 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks @@ -100,7 +61,6 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) \ - -B$(build_prefix)/bin \ -isysroot$(OSX_SDK) -nostdlibinc \ -iwithsysroot/usr/include/c++/v1 \ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks @@ -110,8 +70,8 @@ darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSIO darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION) ifneq ($(build_os),darwin) -darwin_CFLAGS += -mlinker-version=$(LD64_VERSION) -darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION) +darwin_CFLAGS += -mlinker-version=$(LLD_VERSION) +darwin_CXXFLAGS += -mlinker-version=$(LLD_VERSION) darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld endif diff --git a/depends/packages/native_llvm.mk b/depends/packages/native_llvm.mk deleted file mode 100644 index c701147edc..0000000000 --- a/depends/packages/native_llvm.mk +++ /dev/null @@ -1,31 +0,0 @@ -package=native_llvm -$(package)_version=17.0.6 -$(package)_major_version=$(firstword $(subst ., ,$($(package)_version))) -$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) -ifneq (,$(findstring aarch64,$(BUILD))) -$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz -$(package)_sha256_hash=6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a -else -$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-22.04.tar.xz -$(package)_sha256_hash=884ee67d647d77e58740c1e645649e29ae9e8a6fe87c1376be0f3a30f3cc9ab3 -endif - -define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin && \ - mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/dsymutil && \ - cp bin/ld64.lld $($(package)_staging_prefix_dir)/bin/ld64.lld && \ - cp bin/llvm-ar $($(package)_staging_prefix_dir)/bin/llvm-ar && \ - cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ - cp bin/llvm-nm $($(package)_staging_prefix_dir)/bin/llvm-nm && \ - cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/llvm-objdump && \ - cp bin/llvm-ranlib $($(package)_staging_prefix_dir)/bin/llvm-ranlib && \ - cp bin/llvm-strip $($(package)_staging_prefix_dir)/bin/llvm-strip && \ - cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -r lib/clang/$($(package)_major_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include/ -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index ca54093339..01ed0d7a92 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -24,13 +24,3 @@ multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp usdt_linux_packages=systemtap - -darwin_native_packages = - -ifneq ($(build_os),darwin) - -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_packages+= native_llvm -endif - -endif diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 2a97aa351d..10d8ee52eb 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -33,10 +33,10 @@ requests when multiple wallets are in use. ```sh # Get block count from the / endpoint when rpcuser=alice and rpcport=38332 -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: application/json;' localhost:38332/ +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: application/json' localhost:38332/ # Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet -$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: application/json;' localhost:38332/wallet/desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "2.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: application/json' localhost:38332/wallet/desc-wallet ``` @@ -80,7 +80,7 @@ The server recognizes [JSON-RPC v2.0](https://www.jsonrpc.org/specification) req and responds accordingly. A 2.0 request is identified by the presence of `"jsonrpc": "2.0"` in the request body. If that key + value is not present in a request, the legacy JSON-RPC v1.1 protocol is followed instead, which was the only available -protocol in previous releases. +protocol in v27.0 and prior releases. || 1.1 | 2.0 | |-|-|-| @@ -88,7 +88,7 @@ protocol in previous releases. | Response marker | (none) | `"jsonrpc": "2.0"` | | `"error"` and `"result"` fields in response | both present | only one is present | | HTTP codes in response | `200` unless there is any kind of RPC error (invalid parameters, method not found, etc) | Always `200` unless there is an actual HTTP server error (request parsing error, endpoint not found, etc) | -| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field | +| Notifications: requests that get no reply | (not supported) | Supported for requests that exclude the "id" field. Returns HTTP status `204` "No Content" | ## Security diff --git a/doc/dependencies.md b/doc/dependencies.md index 2b9d9128d3..fc574b6164 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -30,7 +30,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | | [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | | [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | -| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.13](https://github.com/bitcoin/bitcoin/pull/29732) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.14](https://github.com/bitcoin/bitcoin/pull/30198) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 7e55962471..eb2bb41aa4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -13,7 +13,6 @@ Developer Notes - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Show sources in debugging](#show-sources-in-debugging) - - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) @@ -386,10 +385,6 @@ ln -s /path/to/project/root/src src 3. Use `debugedit` to modify debug information in the binary. -### Compiling for gprof profiling - -Run configure with the `--enable-gprof` option, then make. - ### `debug.log` If the code is behaving strangely, take a look in the `debug.log` file in the data directory; diff --git a/doc/release-notes-27101.md b/doc/release-notes-27101.md index 8775b59c00..7ce1e9a8c1 100644 --- a/doc/release-notes-27101.md +++ b/doc/release-notes-27101.md @@ -2,8 +2,5 @@ JSON-RPC -------- The JSON-RPC server now recognizes JSON-RPC 2.0 requests and responds with -strict adherence to the specification (https://www.jsonrpc.org/specification): - -- Returning HTTP "204 No Content" responses to JSON-RPC 2.0 notifications instead of full responses. -- Returning HTTP "200 OK" responses in all other cases, rather than 404 responses for unknown methods, 500 responses for invalid parameters, etc. -- Returning either "result" fields or "error" fields in JSON-RPC responses, rather than returning both fields with one field set to null. +strict adherence to the [specification](https://www.jsonrpc.org/specification). +See [JSON-RPC-interface.md](/doc/JSON-RPC-interface.md#json-rpc-11-vs-20) for details.
\ No newline at end of file diff --git a/doc/release-notes-29091-29165.md b/doc/release-notes-29091-29165.md new file mode 100644 index 0000000000..9c9f8e4e50 --- /dev/null +++ b/doc/release-notes-29091-29165.md @@ -0,0 +1,5 @@ +Build +----- + +GCC 11.1 or later, or Clang 15+ or later, +are now required to compile Bitcoin Core. diff --git a/src/Makefile.am b/src/Makefile.am index 87bb2b945c..a69daeae2d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,8 +8,8 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) -AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(SANITIZER_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(CORE_CXXFLAGS) $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_OBJCXXFLAGS = $(AM_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps @@ -292,6 +292,7 @@ BITCOIN_CORE_H = \ util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ + util/bitset.h \ util/bytevectorhash.h \ util/chaintype.h \ util/check.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 7ba0111fa6..2ba72c9e76 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -23,6 +23,7 @@ bench_bench_bitcoin_SOURCES = \ bench/ccoins_caching.cpp \ bench/chacha20.cpp \ bench/checkblock.cpp \ + bench/checkblockindex.cpp \ bench/checkqueue.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 8a638ec690..bde62a1502 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -294,6 +294,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/bech32.cpp \ test/fuzz/bip324.cpp \ test/fuzz/bitdeque.cpp \ + test/fuzz/bitset.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ test/fuzz/blockfilter.cpp \ diff --git a/src/bench/checkblockindex.cpp b/src/bench/checkblockindex.cpp new file mode 100644 index 0000000000..e8a848dbd4 --- /dev/null +++ b/src/bench/checkblockindex.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <test/util/setup_common.h> +#include <validation.h> + +static void CheckBlockIndex(benchmark::Bench& bench) +{ + auto testing_setup{MakeNoLogFileContext<TestChain100Setup>()}; + // Mine some more blocks + testing_setup->mineBlocks(1000); + bench.run([&] { + testing_setup->m_node.chainman->CheckBlockIndex(); + }); +} + + +BENCHMARK(CheckBlockIndex, benchmark::PriorityLevel::HIGH); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 4d2a6f0c2a..ca4434a882 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -151,7 +151,7 @@ int main(int argc, char* argv[]) { LOCK(chainman.GetMutex()); std::cout - << "\t" << "Reindexing: " << std::boolalpha << chainman.m_blockman.m_reindexing.load() << std::noboolalpha << std::endl + << "\t" << "Blockfiles Indexed: " << std::boolalpha << chainman.m_blockman.m_blockfiles_indexed.load() << std::noboolalpha << std::endl << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index cc639d8793..a178abbc93 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -298,7 +298,7 @@ public: } addresses.pushKV("total", total); result.pushKV("addresses_known", std::move(addresses)); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -367,7 +367,7 @@ public: } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } }; @@ -622,7 +622,7 @@ public: } } - return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(UniValue{result}, NullUniValue, /*id=*/1, JSONRPCVersion::V2); } const std::string m_help_doc{ @@ -709,7 +709,7 @@ public: UniValue result(UniValue::VOBJ); result.pushKV("address", address_str); result.pushKV("blocks", reply.get_obj()["result"]); - return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V1_LEGACY); + return JSONRPCReplyObj(std::move(result), NullUniValue, /*id=*/1, JSONRPCVersion::V2); } protected: std::string address_str; @@ -743,8 +743,41 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // 2. port in -rpcconnect (ie following : in ipv4 or ]: in ipv6) // 3. default port for chain uint16_t port{BaseParams().RPCPort()}; - SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); - port = static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", port)); + { + uint16_t rpcconnect_port{0}; + const std::string rpcconnect_str = gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT); + if (!SplitHostPort(rpcconnect_str, rpcconnect_port, host)) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcconnect: %s", rpcconnect_str)); + } else { + if (rpcconnect_port != 0) { + // Use the valid port provided in rpcconnect + port = rpcconnect_port; + } // else, no port was provided in rpcconnect (continue using default one) + } + + if (std::optional<std::string> rpcport_arg = gArgs.GetArg("-rpcport")) { + // -rpcport was specified + const uint16_t rpcport_int{ToIntegral<uint16_t>(rpcport_arg.value()).value_or(0)}; + if (rpcport_int == 0) { + // Uses argument provided as-is + // (rather than value parsed) + // to aid the user in troubleshooting + throw std::runtime_error(strprintf("Invalid port provided in -rpcport: %s", rpcport_arg.value())); + } + + // Use the valid port provided + port = rpcport_int; + + // If there was a valid port provided in rpcconnect, + // rpcconnect_port is non-zero. + if (rpcconnect_port != 0) { + tfm::format(std::cerr, "Warning: Port specified in both -rpcconnect and -rpcport. Using -rpcport %u\n", port); + } + } + } // Obtain event base raii_event_base base = obtain_event_base(); diff --git a/src/init.cpp b/src/init.cpp index 0aac2ac65f..75202c85d3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -598,7 +598,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures every <n> operations. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkaddrman=<n>", strprintf("Run addrman consistency checks every <n> operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkmempool=<n>", strprintf("Run mempool consistency checks every <n> transactions. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -1482,7 +1482,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status); ReadNotificationArgs(args, *node.notifications); - bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), @@ -1523,16 +1522,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!result) { return InitError(util::ErrorString(result)); } - mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); - int64_t descendant_limit_bytes = mempool_opts.limits.descendant_size_vbytes * 40; - if (mempool_opts.max_size_bytes < 0 || mempool_opts.max_size_bytes < descendant_limit_bytes) { - return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0))); - } - LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); + bool do_reindex{args.GetBoolArg("-reindex", false)}; + const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { - node.mempool = std::make_unique<CTxMemPool>(mempool_opts); + bilingual_str mempool_error; + node.mempool = std::make_unique<CTxMemPool>(mempool_opts, mempool_error); + if (!mempool_error.empty()) { + return InitError(mempool_error); + } + LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); ChainstateManager& chainman = *node.chainman; @@ -1558,8 +1558,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = fReindexChainState; + options.wipe_block_tree_db = do_reindex; + options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); @@ -1600,13 +1600,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!fLoaded && !ShutdownRequested(node)) { // first suggest a reindex - if (!options.reindex) { + if (!do_reindex) { bool fRet = uiInterface.ThreadSafeQuestion( error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { - chainman.m_blockman.m_reindexing = true; + do_reindex = true; if (!Assert(node.shutdown)->reset()) { LogPrintf("Internal error: failed to reset shutdown signal.\n"); } @@ -1639,17 +1639,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, chainman.m_blockman.m_reindexing); + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, do_reindex); node.indexes.emplace_back(g_txindex.get()); } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, chainman.m_blockman.m_reindexing); + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, do_reindex); node.indexes.emplace_back(GetBlockFilterIndex(filter_type)); } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, chainman.m_blockman.m_reindexing); + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, do_reindex); node.indexes.emplace_back(g_coin_stats_index.get()); } @@ -1668,7 +1668,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // if pruning, perform the initial blockstore prune // after any wallet rescanning has taken place. if (chainman.m_blockman.IsPruneMode()) { - if (!chainman.m_blockman.m_reindexing) { + if (chainman.m_blockman.m_blockfiles_indexed) { LOCK(cs_main); for (Chainstate* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore…").translated); @@ -1694,7 +1694,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) int chain_active_height = WITH_LOCK(cs_main, return chainman.ActiveChain().Height()); // On first startup, warn on low block storage space - if (!chainman.m_blockman.m_reindexing && !fReindexChainState && chain_active_height <= 1) { + if (!do_reindex && !do_reindex_chainstate && chain_active_height <= 1) { uint64_t assumed_chain_bytes{chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024}; uint64_t additional_bytes_needed{ chainman.m_blockman.IsPruneMode() ? diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 16072b669b..deeba7e318 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -24,7 +24,6 @@ struct BlockManagerOpts { bool fast_prune{false}; const fs::path blocks_dir; Notifications& notifications; - bool reindex{false}; }; } // namespace kernel diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index de5f78494a..076841c3c9 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -33,7 +33,7 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; fs::path datadir; - std::optional<bool> check_block_index{}; + std::optional<int32_t> check_block_index{}; bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; //! If set, it will override the minimum work we will assume exists on some valid chain. std::optional<arith_uint256> minimum_chain_work{}; diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index dd8419a68a..fa76566652 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -33,8 +33,6 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; - if (auto value{args.GetBoolArg("-reindex")}) opts.reindex = *value; - return {}; } } // namespace node diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 4067ccee51..fb62e78138 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -551,7 +551,7 @@ bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_block // Check whether we need to continue reindexing bool fReindexing = false; m_block_tree_db->ReadReindexing(fReindexing); - if (fReindexing) m_reindexing = true; + if (fReindexing) m_blockfiles_indexed = false; return true; } @@ -1182,7 +1182,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile ImportingNow imp{chainman.m_blockman.m_importing}; // -reindex - if (chainman.m_blockman.m_reindexing) { + if (!chainman.m_blockman.m_blockfiles_indexed) { int nFile = 0; // Map of disk positions for blocks with unknown parent (only used for reindex); // parent hash -> child disk position, multiple children can have the same parent. @@ -1205,7 +1205,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile nFile++; } WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); - chainman.m_blockman.m_reindexing = false; + chainman.m_blockman.m_blockfiles_indexed = true; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): chainman.ActiveChainstate().LoadGenesisBlock(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index a501067091..108a08a72b 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -267,18 +267,18 @@ public: explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, - m_interrupt{interrupt}, - m_reindexing{m_opts.reindex} {}; + m_interrupt{interrupt} {} const util::SignalInterrupt& m_interrupt; std::atomic<bool> m_importing{false}; /** - * Tracks if a reindex is currently in progress. Set to true when a reindex - * is requested and false when reindexing completes. Its value is persisted - * in the BlockTreeDB across restarts. + * Whether all blockfiles have been added to the block tree database. + * Normally true, but set to false when a reindex is requested and the + * database is wiped. The value is persisted in the database across restarts + * and will be false until reindexing completes. */ - std::atomic_bool m_reindexing; + std::atomic_bool m_blockfiles_indexed{true}; BlockMap m_block_index GUARDED_BY(cs_main); @@ -359,7 +359,7 @@ public: [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()}; - [[nodiscard]] bool LoadingBlocks() const { return m_importing || m_reindexing; } + [[nodiscard]] bool LoadingBlocks() const { return m_importing || !m_blockfiles_indexed; } /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index d6eb14f513..d7e6176be1 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -45,11 +45,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( .path = chainman.m_options.datadir / "blocks" / "index", .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), .memory_only = options.block_tree_db_in_memory, - .wipe_data = options.reindex, + .wipe_data = options.wipe_block_tree_db, .options = chainman.m_options.block_tree_db}); - if (options.reindex) { + if (options.wipe_block_tree_db) { pblocktree->WriteReindexing(true); + chainman.m_blockman.m_blockfiles_indexed = false; //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { chainman.m_blockman.CleanupBlockRevFiles(); @@ -60,8 +61,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets m_reindexing based on the disk flag! - // From here on, m_reindexing and options.reindex values may be different! + // Note that it also sets m_blockfiles_indexed based on the disk flag! if (!chainman.LoadBlockIndex()) { if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; @@ -84,12 +84,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ImportBlocks after the reindex completes. - if (!chainman.m_blockman.m_reindexing && !chainman.ActiveChainstate().LoadGenesisBlock()) { + if (chainman.m_blockman.m_blockfiles_indexed && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; assert(chainman.m_total_coinstip_cache > 0); @@ -110,7 +110,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( chainstate->InitCoinsDB( /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, - /*should_wipe=*/options.reindex || options.reindex_chainstate); + /*should_wipe=*/options.wipe_chainstate_db); if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); @@ -142,7 +142,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( } } - if (!options.reindex) { + if (!options.wipe_block_tree_db) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { @@ -188,7 +188,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // Load a chain created from a UTXO snapshot, if any exist. bool has_snapshot = chainman.DetectSnapshotChainstate(); - if (has_snapshot && (options.reindex || options.reindex_chainstate)) { + if (has_snapshot && options.wipe_chainstate_db) { LogPrintf("[snapshot] deleting snapshot chainstate due to reindexing\n"); if (!chainman.DeleteSnapshotChainstate()) { return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; @@ -247,7 +247,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); diff --git a/src/node/chainstate.h b/src/node/chainstate.h index a6e9a0331b..bb0c4f2b87 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -22,8 +22,13 @@ struct ChainstateLoadOptions { CTxMemPool* mempool{nullptr}; bool block_tree_db_in_memory{false}; bool coins_db_in_memory{false}; - bool reindex{false}; - bool reindex_chainstate{false}; + // Whether to wipe the block tree database when loading it. If set, this + // will also set a reindexing flag so any existing block data files will be + // scanned and added to the database. + bool wipe_block_tree_db{false}; + // Whether to wipe the chainstate database when loading it. If set, this + // will cause the chainstate database to be rebuilt starting from genesis. + bool wipe_chainstate_db{false}; bool prune{false}; //! Setting require_full_verification to true will require all checks at //! check_level (below) to succeed for loading to succeed. Setting it to diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 1cc126cb05..bc4a815a3e 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -24,7 +24,10 @@ namespace node { util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { - if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + if (auto value{args.GetIntArg("-checkblockindex")}) { + // Interpret bare -checkblockindex argument as 1 instead of 0. + opts.check_block_index = args.GetArg("-checkblockindex")->empty() ? 1 : *value; + } if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; diff --git a/src/psbt.h b/src/psbt.h index 3f74083717..f415d21484 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -237,7 +237,7 @@ struct PSBTInput if (final_script_sig.empty() && final_script_witness.IsNull()) { // Write any partial signatures - for (auto sig_pair : partial_sigs) { + for (const auto& sig_pair : partial_sigs) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_PARTIAL_SIG), Span{sig_pair.second.first}); s << sig_pair.second.second; } diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index d35782189e..87b9f18b33 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -45,6 +45,7 @@ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, request.pushKV("method", strMethod); request.pushKV("params", params); request.pushKV("id", id); + request.pushKV("jsonrpc", "2.0"); return request; } diff --git a/src/rpc/request.h b/src/rpc/request.h index e47f90af86..9968426636 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -17,6 +17,7 @@ enum class JSONRPCVersion { V2 }; +/** JSON-RPC 2.0 request, only used in bitcoin-cli **/ UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(UniValue result, UniValue error, std::optional<UniValue> id, JSONRPCVersion jsonrpc_version); UniValue JSONRPCError(int code, const std::string& message); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 9600258de9..9123bddff4 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -176,7 +176,7 @@ std::string HelpExampleCliNamed(const std::string& methodname, const RPCArgList& std::string HelpExampleRpc(const std::string& methodname, const std::string& args) { return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: application/json;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": [" + args + "]}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& args) @@ -187,7 +187,7 @@ std::string HelpExampleRpcNamed(const std::string& methodname, const RPCArgList& } return "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", " - "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: application/json;' http://127.0.0.1:8332/\n"; + "\"method\": \"" + methodname + "\", \"params\": " + params.write() + "}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"; } // Converts a hex string to a public key if possible diff --git a/src/test/fuzz/bitset.cpp b/src/test/fuzz/bitset.cpp new file mode 100644 index 0000000000..98fcddfb8d --- /dev/null +++ b/src/test/fuzz/bitset.cpp @@ -0,0 +1,316 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> +#include <util/bitset.h> + +#include <bitset> +#include <vector> + +namespace { + +/** Pop the first byte from a Span<const uint8_t>, and return it. */ +uint8_t ReadByte(Span<const uint8_t>& buffer) +{ + if (buffer.empty()) return 0; + uint8_t ret = buffer.front(); + buffer = buffer.subspan(1); + return ret; +} + +/** Perform a simulation fuzz test on BitSet type S. */ +template<typename S> +void TestType(Span<const uint8_t> buffer) +{ + /** This fuzz test's design is based on the assumption that the actual bits stored in the + * bitsets and their simulations do not matter for the purpose of detecting edge cases, thus + * these are taken from a deterministically-seeded RNG instead. To provide some level of + * variation however, pick the seed based on the buffer size and size of the chosen bitset. */ + XoRoShiRo128PlusPlus rng(buffer.size() + 0x10000 * S::Size()); + + using Sim = std::bitset<S::Size()>; + // Up to 4 real BitSets (initially 2). + std::vector<S> real(2); + // Up to 4 std::bitsets with the same corresponding contents. + std::vector<Sim> sim(2); + + /* Compare sim[idx] with real[idx], using all inspector operations. */ + auto compare_fn = [&](unsigned idx) { + /* iterators and operator[] */ + auto it = real[idx].begin(); + unsigned first = S::Size(); + unsigned last = S::Size(); + for (unsigned i = 0; i < S::Size(); ++i) { + bool match = (it != real[idx].end()) && *it == i; + assert(sim[idx][i] == real[idx][i]); + assert(match == real[idx][i]); + assert((it == real[idx].end()) != (it != real[idx].end())); + if (match) { + ++it; + if (first == S::Size()) first = i; + last = i; + } + } + assert(it == real[idx].end()); + assert(!(it != real[idx].end())); + /* Any / None */ + assert(sim[idx].any() == real[idx].Any()); + assert(sim[idx].none() == real[idx].None()); + /* First / Last */ + if (sim[idx].any()) { + assert(first == real[idx].First()); + assert(last == real[idx].Last()); + } + /* Count */ + assert(sim[idx].count() == real[idx].Count()); + }; + + LIMITED_WHILE(buffer.size() > 0, 1000) { + // Read one byte to determine which operation to execute on the BitSets. + int command = ReadByte(buffer) % 64; + // Read another byte that determines which bitsets will be involved. + unsigned args = ReadByte(buffer); + unsigned dest = ((args & 7) * sim.size()) >> 3; + unsigned src = (((args >> 3) & 7) * sim.size()) >> 3; + unsigned aux = (((args >> 6) & 3) * sim.size()) >> 2; + // Args are in range for non-empty sim, or sim is completely empty and will be grown + assert((sim.empty() && dest == 0 && src == 0 && aux == 0) || + (!sim.empty() && dest < sim.size() && src < sim.size() && aux < sim.size())); + + // Pick one operation based on value of command. Not all operations are always applicable. + // Loop through the applicable ones until command reaches 0 (which avoids the need to + // compute the number of applicable commands ahead of time). + while (true) { + if (dest < sim.size() && command-- == 0) { + /* Set() (true) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val); + real[dest].Set(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Reset() */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].reset(val); + real[dest].Reset(val); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Set() (conditional) */ + unsigned val = ReadByte(buffer) % S::Size(); + assert(sim[dest][val] == real[dest][val]); + sim[dest].set(val, args >> 7); + real[dest].Set(val, args >> 7); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct empty. */ + sim.resize(sim.size() + 1); + real.resize(real.size() + 1); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct singleton. */ + unsigned val = ReadByte(buffer) % S::Size(); + std::bitset<S::Size()> newset; + newset[val] = true; + sim.push_back(newset); + real.push_back(S::Singleton(val)); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Make random. */ + compare_fn(dest); + sim[dest].reset(); + real[dest] = S{}; + for (unsigned i = 0; i < S::Size(); ++i) { + if (rng() & 1) { + sim[dest][i] = true; + real[dest].Set(i); + } + } + break; + } else if (dest < sim.size() && command-- == 0) { + /* Assign initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + unsigned r3 = rng() % S::Size(); + compare_fn(dest); + sim[dest].reset(); + real[dest] = {r1, r2, r3}; + sim[dest].set(r1); + sim[dest].set(r2); + sim[dest].set(r3); + break; + } else if (!sim.empty() && command-- == 0) { + /* Destruct. */ + compare_fn(sim.size() - 1); + sim.pop_back(); + real.pop_back(); + break; + } else if (sim.size() < 4 && src < sim.size() && command-- == 0) { + /* Copy construct. */ + sim.emplace_back(sim[src]); + real.emplace_back(real[src]); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* Copy assign. */ + compare_fn(dest); + sim[dest] = sim[src]; + real[dest] = real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* swap() function. */ + swap(sim[dest], sim[src]); + swap(real[dest], real[src]); + break; + } else if (sim.size() < 4 && command-- == 0) { + /* Construct with initializer list. */ + unsigned r1 = rng() % S::Size(); + unsigned r2 = rng() % S::Size(); + sim.emplace_back(); + sim.back().set(r1); + sim.back().set(r2); + real.push_back(S{r1, r2}); + break; + } else if (dest < sim.size() && command-- == 0) { + /* Fill() + copy assign. */ + unsigned len = ReadByte(buffer) % S::Size(); + compare_fn(dest); + sim[dest].reset(); + for (unsigned i = 0; i < len; ++i) sim[dest][i] = true; + real[dest] = S::Fill(len); + break; + } else if (src < sim.size() && command-- == 0) { + /* Iterator copy based compare. */ + unsigned val = ReadByte(buffer) % S::Size(); + /* In a first loop, compare begin..end, and copy to it_copy at some point. */ + auto it = real[src].begin(), it_copy = it; + for (unsigned i = 0; i < S::Size(); ++i) { + if (i == val) it_copy = it; + bool match = (it != real[src].end()) && *it == i; + assert(match == sim[src][i]); + if (match) ++it; + } + assert(it == real[src].end()); + /* Then compare from the copied point again to end. */ + for (unsigned i = val; i < S::Size(); ++i) { + bool match = (it_copy != real[src].end()) && *it_copy == i; + assert(match == sim[src][i]); + if (match) ++it_copy; + } + assert(it_copy == real[src].end()); + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator|= */ + compare_fn(dest); + sim[dest] |= sim[src]; + real[dest] |= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator&= */ + compare_fn(dest); + sim[dest] &= sim[src]; + real[dest] &= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator-= */ + compare_fn(dest); + sim[dest] &= ~sim[src]; + real[dest] -= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && command-- == 0) { + /* operator^= */ + compare_fn(dest); + sim[dest] ^= sim[src]; + real[dest] ^= real[src]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator| */ + compare_fn(dest); + sim[dest] = sim[src] | sim[aux]; + real[dest] = real[src] | real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator& */ + compare_fn(dest); + sim[dest] = sim[src] & sim[aux]; + real[dest] = real[src] & real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator- */ + compare_fn(dest); + sim[dest] = sim[src] & ~sim[aux]; + real[dest] = real[src] - real[aux]; + break; + } else if (src < sim.size() && dest < sim.size() && aux < sim.size() && command-- == 0) { + /* operator^ */ + compare_fn(dest); + sim[dest] = sim[src] ^ sim[aux]; + real[dest] = real[src] ^ real[aux]; + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* IsSupersetOf() and IsSubsetOf() */ + bool is_superset = (sim[aux] & ~sim[src]).none(); + bool is_subset = (sim[src] & ~sim[aux]).none(); + assert(real[src].IsSupersetOf(real[aux]) == is_superset); + assert(real[src].IsSubsetOf(real[aux]) == is_subset); + assert(real[aux].IsSupersetOf(real[src]) == is_subset); + assert(real[aux].IsSubsetOf(real[src]) == is_superset); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* operator== and operator!= */ + assert((sim[src] == sim[aux]) == (real[src] == real[aux])); + assert((sim[src] != sim[aux]) == (real[src] != real[aux])); + break; + } else if (src < sim.size() && aux < sim.size() && command-- == 0) { + /* Overlaps() */ + assert((sim[src] & sim[aux]).any() == real[src].Overlaps(real[aux])); + assert((sim[src] & sim[aux]).any() == real[aux].Overlaps(real[src])); + break; + } + } + } + /* Fully compare the final state. */ + for (unsigned i = 0; i < sim.size(); ++i) { + compare_fn(i); + } +} + +} // namespace + +FUZZ_TARGET(bitset) +{ + unsigned typdat = ReadByte(buffer) % 8; + if (typdat == 0) { + /* 16 bits */ + TestType<bitset_detail::IntBitSet<uint16_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 1>>(buffer); + } else if (typdat == 1) { + /* 32 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 2>>(buffer); + TestType<bitset_detail::IntBitSet<uint32_t>>(buffer); + } else if (typdat == 2) { + /* 48 bits */ + TestType<bitset_detail::MultiIntBitSet<uint16_t, 3>>(buffer); + } else if (typdat == 3) { + /* 64 bits */ + TestType<bitset_detail::IntBitSet<uint64_t>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint64_t, 1>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint16_t, 4>>(buffer); + } else if (typdat == 4) { + /* 96 bits */ + TestType<bitset_detail::MultiIntBitSet<uint32_t, 3>>(buffer); + } else if (typdat == 5) { + /* 128 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 2>>(buffer); + TestType<bitset_detail::MultiIntBitSet<uint32_t, 4>>(buffer); + } else if (typdat == 6) { + /* 192 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 3>>(buffer); + } else if (typdat == 7) { + /* 256 bits */ + TestType<bitset_detail::MultiIntBitSet<uint64_t, 4>>(buffer); + } +} diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index 84f9bb4ad0..3a1663364f 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -7,11 +7,13 @@ #include <test/util/txmempool.h> #include <test/util/mining.h> -#include <node/mini_miner.h> #include <node/miner.h> +#include <node/mini_miner.h> #include <primitives/transaction.h> #include <random.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <deque> #include <vector> @@ -33,7 +35,9 @@ void initialize_miner() FUZZ_TARGET(mini_miner, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); std::vector<COutPoint> outpoints; std::deque<COutPoint> available_coins = g_available_coins; LOCK2(::cs_main, pool.cs); @@ -109,7 +113,9 @@ FUZZ_TARGET(mini_miner, .init = initialize_miner) FUZZ_TARGET(mini_miner_selection, .init = initialize_miner) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CTxMemPool pool{CTxMemPool::Options{}}; + bilingual_str error; + CTxMemPool pool{CTxMemPool::Options{}, error}; + Assert(error.empty()); // Make a copy to preserve determinism. std::deque<COutPoint> available_coins = g_available_coins; std::vector<CTransactionRef> transactions; diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 8304e6fdb8..dd34c465ed 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -43,7 +43,19 @@ FUZZ_TARGET(muhash) }, [&] { // Test that dividing a MuHash by itself brings it back to it's initial state + + // See note about clang + self-assignment in test/uint256_tests.cpp + #if defined(__clang__) + # pragma clang diagnostic push + # pragma clang diagnostic ignored "-Wself-assign-overloaded" + #endif + muhash /= muhash; + + #if defined(__clang__) + # pragma clang diagnostic pop + #endif + muhash.Finalize(out); out2 = uint256S(initial_state_hash); }, diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 6ed47a5e5b..122543ebad 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -15,7 +15,9 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -107,7 +109,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,8 +128,13 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.check_ratio = 1; mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); + bilingual_str error; // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) @@ -149,8 +156,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) auto outpoints_updater = std::make_shared<OutpointsUpdater>(mempool_outpoints); node.validation_signals->RegisterSharedValidationInterface(outpoints_updater); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index 2bf47930f4..791d457710 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -10,6 +10,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstddef> #include <cstdint> @@ -52,7 +54,9 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) CBlockHeaderAndShortTxIDs cmpctblock{*block}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); PartiallyDownloadedBlock pdb{&pool}; // Set of available transactions (mempool or extra_txn) diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 64785948f6..4c7e70e3b0 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -13,6 +13,8 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> +#include <util/translation.h> #include <cstdint> #include <optional> @@ -56,7 +58,9 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) return; } - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) { @@ -90,7 +94,9 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) std::optional<CMutableTransaction> child = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!child) return; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); // Add a bunch of parent-child pairs to the mempool, and remember them. std::vector<CTransaction> mempool_txs; diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index fdad5f802a..85a2663cb9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -15,7 +15,9 @@ #include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/txmempool.h> +#include <util/check.h> #include <util/rbf.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -116,7 +118,7 @@ void MockTime(FuzzedDataProvider& fuzzed_data_provider, const Chainstate& chains SetMockTime(time); } -CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) +std::unique_ptr<CTxMemPool> MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeContext& node) { // Take the default options for tests... CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; @@ -126,7 +128,12 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); // ...and construct a CTxMemPool from it - return CTxMemPool{mempool_opts}; + bilingual_str error; + auto mempool{std::make_unique<CTxMemPool>(std::move(mempool_opts), error)}; + // ... ignore the error since it might be beneficial to fuzz even when the + // mempool size is unreasonably small + Assert(error.empty() || error.original.starts_with("-maxmempool must be at least ")); + return mempool; } void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, bool wtxid_in_mempool) @@ -198,8 +205,8 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN}; SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); @@ -376,8 +383,8 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) } SetMempoolConstraints(*node.args, fuzzed_data_provider); - CTxMemPool tx_pool_{MakeMempool(fuzzed_data_provider, node)}; - MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); + auto tx_pool_{MakeMempool(fuzzed_data_provider, node)}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(tx_pool_.get()); chainstate.SetMempool(&tx_pool); diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 00678742c9..51140ae039 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -13,7 +13,9 @@ #include <test/util/setup_common.h> #include <test/util/txmempool.h> #include <txmempool.h> +#include <util/check.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <cstdint> @@ -40,7 +42,9 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) SetMockTime(ConsumeTime(fuzzed_data_provider)); FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; - CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + bilingual_str error; + CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node), error}; + Assert(error.empty()); auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; chainstate.SetMempool(&pool); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 3300269e1b..c4cf6f8a40 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -14,8 +14,10 @@ #include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/time.h> +#include <util/translation.h> #include <validation.h> #include <versionbits.h> @@ -46,7 +48,9 @@ struct MinerTestingSetup : public TestingSetup { // pointer is not accessed, when the new one should be accessed // instead. m_node.mempool.reset(); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); return *m_node.mempool; } BlockAssembler AssemblerForTest(CTxMemPool& tx_mempool); diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index efa078ca74..0f4c19e197 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(help_example) // test different argument types const RPCArgList& args = {{"foo", "bar"}, {"b", true}, {"n", 1}}; BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", args), "> bitcoin-cli -named test foo=bar b=true n=1\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: application/json;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", args), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"foo\":\"bar\",\"b\":true,\"n\":1}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test shell escape BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"foo", "b'ar"}}), "> bitcoin-cli -named test foo='b'''ar'\n"); @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(help_example) obj_value.pushKV("b", false); obj_value.pushKV("n", 1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", obj_value}}), "> bitcoin-cli -named test name='{\"foo\":\"bar\",\"b\":false,\"n\":1}'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: application/json;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", obj_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":{\"foo\":\"bar\",\"b\":false,\"n\":1}}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test array params UniValue arr_value(UniValue::VARR); @@ -573,7 +573,7 @@ BOOST_AUTO_TEST_CASE(help_example) arr_value.push_back(false); arr_value.push_back(1); BOOST_CHECK_EQUAL(HelpExampleCliNamed("test", {{"name", arr_value}}), "> bitcoin-cli -named test name='[\"bar\",false,1]'\n"); - BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: application/json;' http://127.0.0.1:8332/\n"); + BOOST_CHECK_EQUAL(HelpExampleRpcNamed("test", {{"name", arr_value}}), "> curl --user myusername --data-binary '{\"jsonrpc\": \"2.0\", \"id\": \"curltest\", \"method\": \"test\", \"params\": {\"name\":[\"bar\",false,1]}}' -H 'content-type: application/json' http://127.0.0.1:8332/\n"); // test types don't matter for shell BOOST_CHECK_EQUAL(HelpExampleCliNamed("foo", {{"arg", true}}), HelpExampleCliNamed("foo", {{"arg", "true"}})); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index d872166226..a01ed67d38 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -904,15 +904,15 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vin.clear(); t.vin.resize(2438); // size per input (empty scriptSig): 41 bytes t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(19, 0); // output size: 30 bytes - // tx header: 12 bytes => 48 vbytes - // 2438 inputs: 2438*41 = 99958 bytes => 399832 vbytes - // 1 output: 30 bytes => 120 vbytes - // =============================== - // total: 400000 vbytes + // tx header: 12 bytes => 48 weight units + // 2438 inputs: 2438*41 = 99958 bytes => 399832 weight units + // 1 output: 30 bytes => 120 weight units + // ====================================== + // total: 400000 weight units BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400000); CheckIsStandard(t); - // increase output size by one byte, so we end up with 400004 vbytes + // increase output size by one byte, so we end up with 400004 weight units t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(20, 0); // output size: 31 bytes BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400004); CheckIsNotStandard(t, "tx-size"); diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 5746961550..b7892a2139 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -267,6 +267,22 @@ BOOST_AUTO_TEST_CASE( conversion ) BOOST_AUTO_TEST_CASE( operator_with_self ) { + +/* Clang 16 and earlier detects v -= v and v /= v as self-assignments + to 0 and 1 respectively. + See: https://github.com/llvm/llvm-project/issues/42469 + and the fix in commit c5302325b2a62d77cf13dd16cd5c19141862fed0 . + + This makes some sense for arithmetic classes, but could be considered a bug + elsewhere. Disable the warning here so that the code can be tested, but the + warning should remain on as there will likely always be a better way to + express this. +*/ + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wself-assign-overloaded" +#endif arith_uint256 v = UintToArith256(uint256S("02")); v *= v; BOOST_CHECK(v == UintToArith256(uint256S("04"))); @@ -276,6 +292,9 @@ BOOST_AUTO_TEST_CASE( operator_with_self ) BOOST_CHECK(v == UintToArith256(uint256S("02"))); v -= v; BOOST_CHECK(v == UintToArith256(uint256S("0"))); +#if defined(__clang__) +# pragma clang diagnostic pop +#endif } BOOST_AUTO_TEST_CASE(parse) diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index fd07931716..9e9db32351 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -227,7 +227,9 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto m_node.validation_signals = std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(*m_node.scheduler)); m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); - m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + bilingual_str error{}; + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node), error); + Assert(error.empty()); m_cache_sizes = CalculateCacheSizes(m_args); @@ -236,7 +238,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), - .check_block_index = true, + .check_block_index = 1, .notifications = *m_node.notifications, .signals = m_node.validation_signals.get(), .worker_threads_num = 2, @@ -276,8 +278,8 @@ void ChainTestingSetup::LoadVerifyActivateChainstate() options.mempool = Assert(m_node.mempool.get()); options.block_tree_db_in_memory = m_block_tree_db_in_memory; options.coins_db_in_memory = m_coins_db_in_memory; - options.reindex = chainman.m_blockman.m_reindexing; - options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); + options.wipe_block_tree_db = m_args.GetBoolArg("-reindex", false); + options.wipe_chainstate_db = m_args.GetBoolArg("-reindex", false) || m_args.GetBoolArg("-reindex-chainstate", false); options.prune = chainman.m_blockman.IsPruneMode(); options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c845944d32..10674c07ac 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -16,6 +16,7 @@ #include <policy/settings.h> #include <random.h> #include <reverse_iterator.h> +#include <tinyformat.h> #include <util/check.h> #include <util/feefrac.h> #include <util/moneystr.h> @@ -26,6 +27,7 @@ #include <util/translation.h> #include <validationinterface.h> +#include <algorithm> #include <cmath> #include <numeric> #include <optional> @@ -395,8 +397,19 @@ void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const Options& opts) - : m_opts{opts} +//! Clamp option values and populate the error if options are not valid. +static CTxMemPool::Options&& Flatten(CTxMemPool::Options&& opts, bilingual_str& error) +{ + opts.check_ratio = std::clamp<int>(opts.check_ratio, 0, 1'000'000); + int64_t descendant_limit_bytes = opts.limits.descendant_size_vbytes * 40; + if (opts.max_size_bytes < 0 || opts.max_size_bytes < descendant_limit_bytes) { + error = strprintf(_("-maxmempool must be at least %d MB"), std::ceil(descendant_limit_bytes / 1'000'000.0)); + } + return std::move(opts); +} + +CTxMemPool::CTxMemPool(Options opts, bilingual_str& error) + : m_opts{Flatten(std::move(opts), error)} { } diff --git a/src/txmempool.h b/src/txmempool.h index c9f6cdbfac..52f186f0ff 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -43,6 +43,8 @@ class CChain; class ValidationSignals; +struct bilingual_str; + /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -442,7 +444,7 @@ public: * accepting transactions becomes O(N^2) where N is the number of transactions * in the pool. */ - explicit CTxMemPool(const Options& opts); + explicit CTxMemPool(Options opts, bilingual_str& error); /** * If sanity-checking is turned on, check makes sure the pool is diff --git a/src/util/bitset.h b/src/util/bitset.h new file mode 100644 index 0000000000..6f9e808c37 --- /dev/null +++ b/src/util/bitset.h @@ -0,0 +1,527 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BITSET_H +#define BITCOIN_UTIL_BITSET_H + +#include <util/check.h> + +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <type_traits> + +/* This file provides data types similar to std::bitset, but adds the following functionality: + * + * - Efficient iteration over all set bits (compatible with range-based for loops). + * - Efficient search for the first and last set bit (First() and Last()). + * - Efficient set subtraction: (a - b) implements "a and not b". + * - Efficient non-strict subset/superset testing: IsSubsetOf() and IsSupersetOf(). + * - Efficient set overlap testing: a.Overlaps(b) + * - Efficient construction of set containing 0..N-1 (S::Fill). + * - Efficient construction of a single set (S::Singleton). + * - Construction from initializer lists. + * + * Other differences: + * - BitSet<N> is a bitset that supports at least N elements, but may support more (Size() reports + * the actual number). Because the actual number is unpredictable, there are no operations that + * affect all positions (like std::bitset's operator~, flip(), or all()). + * - Various other unimplemented features. + */ + +namespace bitset_detail { + +/** Count the number of bits set in an unsigned integer type. */ +template<typename I> +unsigned inline constexpr PopCount(I v) +{ + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + constexpr auto BITS = std::numeric_limits<I>::digits; + // Algorithms from https://en.wikipedia.org/wiki/Hamming_weight#Efficient_implementation. + // These seem to be faster than std::popcount when compiling for non-SSE4 on x86_64. + if constexpr (BITS <= 32) { + v -= (v >> 1) & 0x55555555; + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + v = (v + (v >> 4)) & 0x0f0f0f0f; + if constexpr (BITS > 8) v += v >> 8; + if constexpr (BITS > 16) v += v >> 16; + return v & 0x3f; + } else { + static_assert(BITS <= 64); + v -= (v >> 1) & 0x5555555555555555; + v = (v & 0x3333333333333333) + ((v >> 2) & 0x3333333333333333); + v = (v + (v >> 4)) & 0x0f0f0f0f0f0f0f0f; + return (v * uint64_t{0x0101010101010101}) >> 56; + } +} + +/** A bitset implementation backed by a single integer of type I. */ +template<typename I> +class IntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + /** The maximum number of bits this bitset supports. */ + static constexpr unsigned MAX_SIZE = std::numeric_limits<I>::digits; + /** Integer whose bits represent this bitset. */ + I m_val; + /** Internal constructor with a given integer as contents. */ + IntBitSet(I val) noexcept : m_val{val} {} + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class IntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class IntBitSet; + I m_val; /**< The original integer's remaining bits. */ + unsigned m_pos; /** Last reported 1 position (if m_pos != 0). */ + constexpr Iterator(I val) noexcept : m_val(val), m_pos(0) + { + if (m_val != 0) m_pos = std::countr_zero(m_val); + } + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + constexpr friend bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_val == 0; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_val != 0); + m_val &= m_val - I{1U}; + if (m_val != 0) m_pos = std::countr_zero(m_val); + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_val != 0); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr IntBitSet() noexcept : m_val{0} {} + /** Copy construct a bitset. */ + constexpr IntBitSet(const IntBitSet&) noexcept = default; + /** Construct from a list of values. */ + constexpr IntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val(0) + { + for (auto pos : ilist) Set(pos); + } + /** Copy assign a bitset. */ + constexpr IntBitSet& operator=(const IntBitSet&) noexcept = default; + /** Assign from a list of positions (which will be made true, all others false). */ + constexpr IntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val = 0; + for (auto pos : ilist) Set(pos); + return *this; + } + /** Construct a bitset with the singleton i. */ + static constexpr IntBitSet Singleton(unsigned i) noexcept + { + Assume(i < MAX_SIZE); + return IntBitSet(I(1U) << i); + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr IntBitSet Fill(unsigned count) noexcept + { + IntBitSet ret; + Assume(count <= MAX_SIZE); + if (count) ret.m_val = I(~I{0}) >> (MAX_SIZE - count); + return ret; + } + /** Set a bit to 1. */ + constexpr void Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val |= I{1U} << pos; + } + /** Set a bit to the specified value. */ + constexpr void Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val = (m_val & ~I(I{1U} << pos)) | (I(val) << pos); + } + /** Set a bit to 0. */ + constexpr void Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val &= ~I(I{1U} << pos); + } + /** Retrieve a bit at the given position. */ + constexpr bool operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val >> pos) & 1U; + } + /** Compute the number of 1 bits in the bitset. */ + constexpr unsigned Count() const noexcept { return PopCount(m_val); } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Check if all bits are 0. */ + constexpr bool None() const noexcept { return m_val == 0; } + /** Check if any bits are 1. */ + constexpr bool Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + constexpr Iterator begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + constexpr IteratorEnd end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + constexpr unsigned First() const noexcept + { + Assume(m_val != 0); + return std::countr_zero(m_val); + } + /** Find the last element (requires Any()). */ + constexpr unsigned Last() const noexcept + { + Assume(m_val != 0); + return std::bit_width(m_val) - 1; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr IntBitSet& operator|=(const IntBitSet& a) noexcept { m_val |= a.m_val; return *this; } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr IntBitSet& operator&=(const IntBitSet& a) noexcept { m_val &= a.m_val; return *this; } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr IntBitSet& operator-=(const IntBitSet& a) noexcept { m_val &= ~a.m_val; return *this; } + /** Set this object's bits to be the binary XOR between respective bits from this as a. */ + constexpr IntBitSet& operator^=(const IntBitSet& a) noexcept { m_val ^= a.m_val; return *this; } + /** Check if the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const IntBitSet& a) const noexcept { return m_val & a.m_val; } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr IntBitSet operator&(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & b.m_val); } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr IntBitSet operator|(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val | b.m_val); } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr IntBitSet operator-(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val & ~b.m_val); } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr IntBitSet operator^(const IntBitSet& a, const IntBitSet& b) noexcept { return I(a.m_val ^ b.m_val); } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const IntBitSet& a, const IntBitSet& b) noexcept = default; + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const IntBitSet& a) const noexcept { return (a.m_val & ~m_val) == 0; } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const IntBitSet& a) const noexcept { return (m_val & ~a.m_val) == 0; } + /** Swap two bitsets. */ + friend constexpr void swap(IntBitSet& a, IntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +/** A bitset implementation backed by N integers of type I. */ +template<typename I, unsigned N> +class MultiIntBitSet +{ + // Only binary, unsigned, integer, types allowed. + static_assert(std::is_integral_v<I> && std::is_unsigned_v<I> && std::numeric_limits<I>::radix == 2); + // Cannot be empty. + static_assert(N > 0); + /** The number of bits per integer. */ + static constexpr unsigned LIMB_BITS = std::numeric_limits<I>::digits; + /** Number of elements this set type supports. */ + static constexpr unsigned MAX_SIZE = LIMB_BITS * N; + // No overflow allowed here. + static_assert(MAX_SIZE / LIMB_BITS == N); + /** Array whose member integers store the bits of the set. */ + std::array<I, N> m_val; + /** Dummy type to return using end(). Only used for comparing with Iterator. */ + class IteratorEnd + { + friend class MultiIntBitSet; + constexpr IteratorEnd() = default; + public: + constexpr IteratorEnd(const IteratorEnd&) = default; + }; + /** Iterator type returned by begin(), which efficiently iterates all 1 positions. */ + class Iterator + { + friend class MultiIntBitSet; + const std::array<I, N>* m_ptr; /**< Pointer to array to fetch bits from. */ + I m_val; /**< The remaining bits of (*m_ptr)[m_idx]. */ + unsigned m_pos; /**< The last reported position. */ + unsigned m_idx; /**< The index in *m_ptr currently being iterated over. */ + constexpr Iterator(const std::array<I, N>& ref) noexcept : m_ptr(&ref), m_idx(0) + { + do { + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + ++m_idx; + } while(m_idx < N); + } + + public: + /** Do not allow external code to construct an Iterator. */ + Iterator() = delete; + // Copying is allowed. + constexpr Iterator(const Iterator&) noexcept = default; + constexpr Iterator& operator=(const Iterator&) noexcept = default; + /** Test whether we are done (can only compare with IteratorEnd). */ + friend constexpr bool operator==(const Iterator& a, const IteratorEnd&) noexcept + { + return a.m_idx == N; + } + /** Progress to the next 1 bit (only if != IteratorEnd). */ + constexpr Iterator& operator++() noexcept + { + Assume(m_idx < N); + m_val &= m_val - I{1U}; + if (m_val == 0) { + while (true) { + ++m_idx; + if (m_idx == N) break; + m_val = (*m_ptr)[m_idx]; + if (m_val) { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + break; + } + } + } else { + m_pos = std::countr_zero(m_val) + m_idx * LIMB_BITS; + } + return *this; + } + /** Get the current bit position (only if != IteratorEnd). */ + constexpr unsigned operator*() const noexcept + { + Assume(m_idx < N); + return m_pos; + } + }; + +public: + /** Construct an all-zero bitset. */ + constexpr MultiIntBitSet() noexcept : m_val{} {} + /** Copy construct a bitset. */ + constexpr MultiIntBitSet(const MultiIntBitSet&) noexcept = default; + /** Copy assign a bitset. */ + constexpr MultiIntBitSet& operator=(const MultiIntBitSet&) noexcept = default; + /** Set a bit to 1. */ + void constexpr Set(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] |= I{1U} << (pos % LIMB_BITS); + } + /** Set a bit to the specified value. */ + void constexpr Set(unsigned pos, bool val) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] = (m_val[pos / LIMB_BITS] & ~I(I{1U} << (pos % LIMB_BITS))) | + (I{val} << (pos % LIMB_BITS)); + } + /** Construct a bitset from a list of values. */ + constexpr MultiIntBitSet(std::initializer_list<unsigned> ilist) noexcept : m_val{} + { + for (auto pos : ilist) Set(pos); + } + /** Set a bitset to a list of values. */ + constexpr MultiIntBitSet& operator=(std::initializer_list<unsigned> ilist) noexcept + { + m_val.fill(0); + for (auto pos : ilist) Set(pos); + return *this; + } + /** Set a bit to 0. */ + void constexpr Reset(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + m_val[pos / LIMB_BITS] &= ~I(I{1U} << (pos % LIMB_BITS)); + } + /** Retrieve a bit at the given position. */ + bool constexpr operator[](unsigned pos) const noexcept + { + Assume(pos < MAX_SIZE); + return (m_val[pos / LIMB_BITS] >> (pos % LIMB_BITS)) & 1U; + } + /** Construct a bitset with the singleton pos. */ + static constexpr MultiIntBitSet Singleton(unsigned pos) noexcept + { + Assume(pos < MAX_SIZE); + MultiIntBitSet ret; + ret.m_val[pos / LIMB_BITS] = I{1U} << (pos % LIMB_BITS); + return ret; + } + /** Construct a bitset with bits 0..count-1 (inclusive) set to 1. */ + static constexpr MultiIntBitSet Fill(unsigned count) noexcept + { + Assume(count <= MAX_SIZE); + MultiIntBitSet ret; + if (count) { + unsigned i = 0; + while (count > LIMB_BITS) { + ret.m_val[i++] = ~I{0}; + count -= LIMB_BITS; + } + ret.m_val[i] = I(~I{0}) >> (LIMB_BITS - count); + } + return ret; + } + /** Return the number of bits that this object holds. */ + static constexpr unsigned Size() noexcept { return MAX_SIZE; } + /** Compute the number of 1 bits in the bitset. */ + unsigned constexpr Count() const noexcept + { + unsigned ret{0}; + for (I v : m_val) ret += PopCount(v); + return ret; + } + /** Check if all bits are 0. */ + bool constexpr None() const noexcept + { + for (auto v : m_val) { + if (v != 0) return false; + } + return true; + } + /** Check if any bits are 1. */ + bool constexpr Any() const noexcept { return !None(); } + /** Return an object that iterates over all 1 bits (++ and * only allowed when != end()). */ + Iterator constexpr begin() const noexcept { return Iterator(m_val); } + /** Return a dummy object to compare Iterators with. */ + IteratorEnd constexpr end() const noexcept { return IteratorEnd(); } + /** Find the first element (requires Any()). */ + unsigned constexpr First() const noexcept + { + unsigned p = 0; + while (m_val[p] == 0) { + ++p; + Assume(p < N); + } + return std::countr_zero(m_val[p]) + p * LIMB_BITS; + } + /** Find the last element (requires Any()). */ + unsigned constexpr Last() const noexcept + { + unsigned p = N - 1; + while (m_val[p] == 0) { + Assume(p > 0); + --p; + } + return std::bit_width(m_val[p]) - 1 + p * LIMB_BITS; + } + /** Set this object's bits to be the binary OR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator|=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] |= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND between respective bits from this and a. */ + constexpr MultiIntBitSet& operator&=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary AND NOT between respective bits from this and a. */ + constexpr MultiIntBitSet& operator-=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] &= ~a.m_val[i]; + } + return *this; + } + /** Set this object's bits to be the binary XOR between respective bits from this and a. */ + constexpr MultiIntBitSet& operator^=(const MultiIntBitSet& a) noexcept + { + for (unsigned i = 0; i < N; ++i) { + m_val[i] ^= a.m_val[i]; + } + return *this; + } + /** Check whether the intersection between two sets is non-empty. */ + constexpr bool Overlaps(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & a.m_val[i]) return true; + } + return false; + } + /** Return an object with the binary AND between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator&(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & b.m_val[i]; + } + return r; + } + /** Return an object with the binary OR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator|(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] | b.m_val[i]; + } + return r; + } + /** Return an object with the binary AND NOT between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator-(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] & ~b.m_val[i]; + } + return r; + } + /** Return an object with the binary XOR between respective bits from a and b. */ + friend constexpr MultiIntBitSet operator^(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept + { + MultiIntBitSet r; + for (unsigned i = 0; i < N; ++i) { + r.m_val[i] = a.m_val[i] ^ b.m_val[i]; + } + return r; + } + /** Check if bitset a is a superset of bitset b (= every 1 bit in b is also in a). */ + constexpr bool IsSupersetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (a.m_val[i] & ~m_val[i]) return false; + } + return true; + } + /** Check if bitset a is a subset of bitset b (= every 1 bit in a is also in b). */ + constexpr bool IsSubsetOf(const MultiIntBitSet& a) const noexcept + { + for (unsigned i = 0; i < N; ++i) { + if (m_val[i] & ~a.m_val[i]) return false; + } + return true; + } + /** Check if bitset a and bitset b are identical. */ + friend constexpr bool operator==(const MultiIntBitSet& a, const MultiIntBitSet& b) noexcept = default; + /** Swap two bitsets. */ + friend constexpr void swap(MultiIntBitSet& a, MultiIntBitSet& b) noexcept { std::swap(a.m_val, b.m_val); } +}; + +} // namespace bitset_detail + +// BitSet dispatches to IntBitSet or MultiIntBitSet as appropriate for the requested minimum number +// of bits. Use IntBitSet up to 32-bit, or up to 64-bit on 64-bit platforms; above that, use a +// MultiIntBitSet of size_t. +template<unsigned BITS> +using BitSet = std::conditional_t<(BITS <= 32), bitset_detail::IntBitSet<uint32_t>, + std::conditional_t<(BITS <= std::numeric_limits<size_t>::digits), bitset_detail::IntBitSet<size_t>, + bitset_detail::MultiIntBitSet<size_t, (BITS + std::numeric_limits<size_t>::digits - 1) / std::numeric_limits<size_t>::digits>>>; + +#endif // BITCOIN_UTIL_BITSET_H diff --git a/src/validation.cpp b/src/validation.cpp index 8c766caca1..e066ee16cb 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2700,7 +2700,7 @@ bool Chainstate::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); - if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !m_chainman.m_blockman.m_reindexing) { + if (m_blockman.IsPruneMode() && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && m_chainman.m_blockman.m_blockfiles_indexed) { // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based int last_prune{m_chain.Height()}; // last height we can prune @@ -3313,10 +3313,10 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* return true; } -static SynchronizationState GetSynchronizationState(bool init, bool reindexing) +static SynchronizationState GetSynchronizationState(bool init, bool blockfiles_indexed) { if (!init) return SynchronizationState::POST_INIT; - if (reindexing) return SynchronizationState::INIT_REINDEX; + if (!blockfiles_indexed) return SynchronizationState::INIT_REINDEX; return SynchronizationState::INIT_DOWNLOAD; } @@ -3338,7 +3338,7 @@ static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) } // Send block tip changed notifications without cs_main if (fNotify) { - chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_reindexing), pindexHeader->nHeight, pindexHeader->nTime, false); + chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload, chainman.m_blockman.m_blockfiles_indexed), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3457,7 +3457,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< } // Always notify the UI if a new block tip was connected - if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_reindexing), *pindexNewTip))) { + if (kernel::IsInterrupted(m_chainman.GetNotifications().blockTip(GetSynchronizationState(still_in_ibd, m_chainman.m_blockman.m_blockfiles_indexed), *pindexNewTip))) { // Just breaking and returning success for now. This could // be changed to bubble up the kernel::Interrupted value to // the caller so the caller could distinguish between @@ -3683,7 +3683,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // parameter indicating the source of the tip change so hooks can // distinguish user-initiated invalidateblock changes from other // changes. - (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_reindexing), *to_mark_failed->pprev); + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload(), m_chainman.m_blockman.m_blockfiles_indexed), *to_mark_failed->pprev); } return true; } @@ -4322,7 +4322,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = IsInitialBlockDownload(); - GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_reindexing), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download, m_blockman.m_blockfiles_indexed), height, timestamp, /*presync=*/true); if (initial_download) { int64_t blocks_left{(NodeClock::now() - NodeSeconds{std::chrono::seconds{timestamp}}) / GetConsensus().PowTargetSpacing()}; blocks_left = std::max<int64_t>(0, blocks_left); @@ -4849,8 +4849,7 @@ bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases - bool needs_init = m_blockman.m_reindexing; - if (!m_blockman.m_reindexing) { + if (m_blockman.m_blockfiles_indexed) { bool ret{m_blockman.LoadBlockIndexDB(SnapshotBlockhash())}; if (!ret) return false; @@ -4881,18 +4880,6 @@ bool ChainstateManager::LoadBlockIndex() if (pindex->IsValid(BLOCK_VALID_TREE) && (m_best_header == nullptr || CBlockIndexWorkComparator()(m_best_header, pindex))) m_best_header = pindex; } - - needs_init = m_blockman.m_block_index.empty(); - } - - if (needs_init) { - // Everything here is for *new* reindex/DBs. Thus, though - // LoadBlockIndexDB may have set m_reindexing if we shut down - // mid-reindex previously, we don't check m_reindexing and - // instead only check it prior to LoadBlockIndexDB to set - // needs_init. - - LogPrintf("Initializing databases...\n"); } return true; } @@ -5033,7 +5020,7 @@ void ChainstateManager::LoadExternalBlockFile( } } - if (m_blockman.IsPruneMode() && !m_blockman.m_reindexing && pblock) { + if (m_blockman.IsPruneMode() && m_blockman.m_blockfiles_indexed && pblock) { // must update the tip for pruning to work while importing with -loadblock. // this is a tradeoff to conserve disk space at the expense of time // spent updating the tip to be able to prune. @@ -5105,6 +5092,14 @@ void ChainstateManager::LoadExternalBlockFile( LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } +bool ChainstateManager::ShouldCheckBlockIndex() const +{ + // Assert to verify Flatten() has been called. + if (!*Assert(m_options.check_block_index)) return false; + if (GetRand(*m_options.check_block_index) >= 1) return false; + return true; +} + void ChainstateManager::CheckBlockIndex() { if (!ShouldCheckBlockIndex()) { @@ -5121,19 +5116,28 @@ void ChainstateManager::CheckBlockIndex() return; } - // Build forward-pointing map of the entire block tree. + // Build forward-pointing data structure for the entire block tree. + // For performance reasons, indexes of the best header chain are stored in a vector (within CChain). + // All remaining blocks are stored in a multimap. + // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that + // are not yet validated. + CChain best_hdr_chain; + assert(m_best_header); + best_hdr_chain.SetTip(*m_best_header); + std::multimap<CBlockIndex*,CBlockIndex*> forward; for (auto& [_, block_index] : m_blockman.m_block_index) { - forward.emplace(block_index.pprev, &block_index); + // Only save indexes in forward that are not part of the best header chain. + if (!best_hdr_chain.Contains(&block_index)) { + // Only genesis, which must be part of the best header chain, can have a nullptr parent. + assert(block_index.pprev); + forward.emplace(block_index.pprev, &block_index); + } } + assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size()); - assert(forward.size() == m_blockman.m_block_index.size()); - - std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(nullptr); - CBlockIndex *pindex = rangeGenesis.first->second; - rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. - + CBlockIndex* pindex = best_hdr_chain[0]; + assert(pindex); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. @@ -5345,14 +5349,21 @@ void ChainstateManager::CheckBlockIndex() // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. - // Try descending into the first subnode. + + // Try descending into the first subnode. Always process forks first and the best header chain after. snap_update_firsts(); std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { - // A subnode was found. + // A subnode not part of the best header chain was found. pindex = range.first->second; nHeight++; continue; + } else if (best_hdr_chain.Contains(pindex)) { + // Descend further into best header chain. + nHeight++; + pindex = best_hdr_chain[nHeight]; + if (!pindex) break; // we are finished, since the best header chain is always processed last + continue; } // This is a leaf node. // Move upwards until we reach a node of which we have not yet visited the last child. @@ -5378,9 +5389,15 @@ void ChainstateManager::CheckBlockIndex() // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { - // Move to the sibling. + // Move to a sibling not part of the best header chain. pindex = rangePar.first->second; break; + } else if (pindexPar == best_hdr_chain[nHeight - 1]) { + // Move to pindex's sibling on the best-chain, if it has one. + pindex = best_hdr_chain[nHeight]; + // There will not be a next block if (and only if) parent block is the best header. + assert((pindex == nullptr) == (pindexPar == best_hdr_chain.Tip())); + break; } else { // Move up further. pindex = pindexPar; @@ -5390,8 +5407,8 @@ void ChainstateManager::CheckBlockIndex() } } - // Check that we actually traversed the entire map. - assert(nNodes == forward.size()); + // Check that we actually traversed the entire block index. + assert(nNodes == forward.size() + best_hdr_chain.Height() + 1); } std::string Chainstate::ToString() diff --git a/src/validation.h b/src/validation.h index ea6b6cad7e..ab7891539a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -938,7 +938,7 @@ public: const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } - bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + bool ShouldCheckBlockIndex() const; const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } kernel::Notifications& GetNotifications() const { return m_options.notifications; }; diff --git a/test/functional/feature_framework_miniwallet.py b/test/functional/feature_framework_miniwallet.py new file mode 100755 index 0000000000..f108289018 --- /dev/null +++ b/test/functional/feature_framework_miniwallet.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test MiniWallet.""" +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_greater_than_or_equal, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + + +class FeatureFrameworkMiniWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def test_tx_padding(self): + """Verify that MiniWallet's transaction padding (`target_weight` parameter) + works accurately enough (i.e. at most 3 WUs higher) with all modes.""" + for mode_name, wallet in self.wallets: + self.log.info(f"Test tx padding with MiniWallet mode {mode_name}...") + utxo = wallet.get_utxo(mark_as_spent=False) + for target_weight in [1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 4000000, + 989, 2001, 4337, 13371, 23219, 49153, 102035, 223419, 3999989]: + tx = wallet.create_self_transfer(utxo_to_spend=utxo, target_weight=target_weight)["tx"] + self.log.debug(f"-> target weight: {target_weight}, actual weight: {tx.get_weight()}") + assert_greater_than_or_equal(tx.get_weight(), target_weight) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) + + def run_test(self): + node = self.nodes[0] + self.wallets = [ + ("ADDRESS_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.ADDRESS_OP_TRUE)), + ("RAW_OP_TRUE", MiniWallet(node, mode=MiniWalletMode.RAW_OP_TRUE)), + ("RAW_P2PK", MiniWallet(node, mode=MiniWalletMode.RAW_P2PK)), + ] + for _, wallet in self.wallets: + self.generate(wallet, 10) + self.generate(wallet, COINBASE_MATURITY) + + self.test_tx_padding() + + +if __name__ == '__main__': + FeatureFrameworkMiniWalletTest().main() diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index f0f32a61ab..835cd0c5cf 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -73,6 +73,25 @@ class ReindexTest(BitcoinTestFramework): # All blocks should be accepted and processed. assert_equal(self.nodes[0].getblockcount(), 12) + def continue_reindex_after_shutdown(self): + node = self.nodes[0] + self.generate(node, 1500) + + # Restart node with reindex and stop reindex as soon as it starts reindexing + self.log.info("Restarting node while reindexing..") + node.stop_node() + with node.busy_wait_for_debug_log([b'initload thread start']): + node.start(['-blockfilterindex', '-reindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + + # Start node without the reindex flag and verify it does not wipe the indexes data again + db_path = node.chain_path / 'indexes' / 'blockfilter' / 'basic' / 'db' + with node.assert_debug_log(expected_msgs=[f'Opening LevelDB in {db_path}'], unexpected_msgs=[f'Wiping LevelDB in {db_path}']): + node.start(['-blockfilterindex']) + node.wait_for_rpc_connection(wait_for_import=False) + node.stop_node() + def run_test(self): self.reindex(False) self.reindex(True) @@ -80,6 +99,7 @@ class ReindexTest(BitcoinTestFramework): self.reindex(True) self.out_of_order() + self.continue_reindex_after_shutdown() if __name__ == '__main__': diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 83bb5121e5..a6628dcbf3 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -8,6 +8,7 @@ from decimal import Decimal import re from test_framework.blocktools import COINBASE_MATURITY +from test_framework.netutil import test_ipv6_local from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -15,6 +16,7 @@ from test_framework.util import ( assert_raises_process_error, assert_raises_rpc_error, get_auth_cookie, + rpc_port, ) import time @@ -107,6 +109,53 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) + self.log.info("Test handling of invalid ports in rpcconnect") + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:notaport", self.nodes[0].cli("-rpcconnect=127.0.0.1:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:-1", self.nodes[0].cli("-rpcconnect=127.0.0.1:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:0", self.nodes[0].cli("-rpcconnect=127.0.0.1:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: 127.0.0.1:65536", self.nodes[0].cli("-rpcconnect=127.0.0.1:65536").echo) + + self.log.info("Checking for IPv6") + have_ipv6 = test_ipv6_local() + if not have_ipv6: + self.log.info("Skipping IPv6 tests") + + if have_ipv6: + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:notaport", self.nodes[0].cli("-rpcconnect=[::1]:notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:-1", self.nodes[0].cli("-rpcconnect=[::1]:-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:0", self.nodes[0].cli("-rpcconnect=[::1]:0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcconnect: [::1]:65536", self.nodes[0].cli("-rpcconnect=[::1]:65536").echo) + + self.log.info("Test handling of invalid ports in rpcport") + assert_raises_process_error(1, "Invalid port provided in -rpcport: notaport", self.nodes[0].cli("-rpcport=notaport").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: -1", self.nodes[0].cli("-rpcport=-1").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 0", self.nodes[0].cli("-rpcport=0").echo) + assert_raises_process_error(1, "Invalid port provided in -rpcport: 65536", self.nodes[0].cli("-rpcport=65536").echo) + + self.log.info("Test port usage preferences") + node_rpc_port = rpc_port(self.nodes[0].index) + # Prevent bitcoin-cli from using existing rpcport in conf + conf_rpcport = "rpcport=" + str(node_rpc_port) + self.nodes[0].replace_in_config([(conf_rpcport, "#" + conf_rpcport)]) + # prefer rpcport over rpcconnect + assert_raises_process_error(1, "Could not connect to the server 127.0.0.1:1", self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}", "-rpcport=1").echo) + if have_ipv6: + assert_raises_process_error(1, "Could not connect to the server ::1:1", self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}", "-rpcport=1").echo) + + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=127.0.0.1:18999", f'-rpcport={node_rpc_port}').getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli("-rpcconnect=[::1]:18999", f'-rpcport={node_rpc_port}').getblockcount()) + + # prefer rpcconnect port over default + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=127.0.0.1:{node_rpc_port}").getblockcount()) + if have_ipv6: + assert_equal(BLOCKS, self.nodes[0].cli(f"-rpcconnect=[::1]:{node_rpc_port}").getblockcount()) + + # prefer rpcport over default + assert_equal(BLOCKS, self.nodes[0].cli(f'-rpcport={node_rpc_port}').getblockcount()) + # Re-enable rpcport in conf if present + self.nodes[0].replace_in_config([("#" + conf_rpcport, conf_rpcport)]) + self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b08ca42796..6c1855c400 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -14,7 +14,6 @@ from typing import Optional import subprocess -RPC_INVALID_ADDRESS_OR_KEY = -5 RPC_INVALID_PARAMETER = -8 RPC_METHOD_NOT_FOUND = -32601 RPC_INVALID_REQUEST = -32600 diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index d46924f4ce..49a0a32c45 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -59,7 +59,7 @@ class MempoolLimitTest(BitcoinTestFramework): mempoolmin_feerate = node.getmempoolinfo()["mempoolminfee"] tx_A = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (A_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=A_weight, utxo_to_spend=rbf_utxo, confirmed_only=True @@ -77,7 +77,7 @@ class MempoolLimitTest(BitcoinTestFramework): non_cpfp_carveout_weight = 40001 # EXTRA_DESCENDANT_TX_SIZE_LIMIT + 1 tx_C = self.wallet.create_self_transfer( target_weight=non_cpfp_carveout_weight, - fee = (mempoolmin_feerate / 1000) * (non_cpfp_carveout_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, utxo_to_spend=tx_B["new_utxo"], confirmed_only=True ) @@ -109,7 +109,7 @@ class MempoolLimitTest(BitcoinTestFramework): # happen in the middle of package evaluation, as it can invalidate the coins cache. mempool_evicted_tx = self.wallet.send_self_transfer( from_node=node, - fee=(mempoolmin_feerate / 1000) * (evicted_weight // 4) + Decimal('0.000001'), + fee_rate=mempoolmin_feerate, target_weight=evicted_weight, confirmed_only=True ) @@ -135,11 +135,11 @@ class MempoolLimitTest(BitcoinTestFramework): parent_weight = 100000 num_big_parents = 3 assert_greater_than(parent_weight * num_big_parents, current_info["maxmempool"] - current_info["bytes"]) - parent_fee = (100 * mempoolmin_feerate / 1000) * (parent_weight // 4) + parent_feerate = 100 * mempoolmin_feerate big_parent_txids = [] for i in range(num_big_parents): - parent = self.wallet.create_self_transfer(fee=parent_fee, target_weight=parent_weight, confirmed_only=True) + parent = self.wallet.create_self_transfer(fee_rate=parent_feerate, target_weight=parent_weight, confirmed_only=True) parent_utxos.append(parent["new_utxo"]) package_hex.append(parent["hex"]) big_parent_txids.append(parent["txid"]) @@ -314,18 +314,20 @@ class MempoolLimitTest(BitcoinTestFramework): target_weight_each = 200000 assert_greater_than(target_weight_each * 2, node.getmempoolinfo()["maxmempool"] - node.getmempoolinfo()["bytes"]) # Should be a true CPFP: parent's feerate is just below mempool min feerate - parent_fee = (mempoolmin_feerate / 1000) * (target_weight_each // 4) - Decimal("0.00001") + parent_feerate = mempoolmin_feerate - Decimal("0.000001") # 0.1 sats/vbyte below min feerate # Parent + child is above mempool minimum feerate - child_fee = (worst_feerate_btcvb) * (target_weight_each // 4) - Decimal("0.00001") + child_feerate = (worst_feerate_btcvb * 1000) - Decimal("0.000001") # 0.1 sats/vbyte below worst feerate # However, when eviction is triggered, these transactions should be at the bottom. # This assertion assumes parent and child are the same size. miniwallet.rescan_utxos() - tx_parent_just_below = miniwallet.create_self_transfer(fee=parent_fee, target_weight=target_weight_each) - tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee=child_fee, target_weight=target_weight_each) + tx_parent_just_below = miniwallet.create_self_transfer(fee_rate=parent_feerate, target_weight=target_weight_each) + tx_child_just_above = miniwallet.create_self_transfer(utxo_to_spend=tx_parent_just_below["new_utxo"], fee_rate=child_feerate, target_weight=target_weight_each) # This package ranks below the lowest descendant package in the mempool - assert_greater_than(worst_feerate_btcvb, (parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize())) - assert_greater_than(mempoolmin_feerate, (parent_fee) / (tx_parent_just_below["tx"].get_vsize())) - assert_greater_than((parent_fee + child_fee) / (tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize()), mempoolmin_feerate / 1000) + package_fee = tx_parent_just_below["fee"] + tx_child_just_above["fee"] + package_vsize = tx_parent_just_below["tx"].get_vsize() + tx_child_just_above["tx"].get_vsize() + assert_greater_than(worst_feerate_btcvb, package_fee / package_vsize) + assert_greater_than(mempoolmin_feerate, tx_parent_just_below["fee"] / (tx_parent_just_below["tx"].get_vsize())) + assert_greater_than(package_fee / package_vsize, mempoolmin_feerate / 1000) res = node.submitpackage([tx_parent_just_below["hex"], tx_child_just_above["hex"]]) for wtxid in [tx_parent_just_below["wtxid"], tx_child_just_above["wtxid"]]: assert_equal(res["tx-results"][wtxid]["error"], "mempool full") diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 800ec4b11e..f974a05f7b 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -599,6 +599,8 @@ class RawTransactionsTest(BitcoinTestFramework): rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) self.log.debug(rawTxPartialSigned2) assert_equal(rawTxPartialSigned2['complete'], False) # node2 only has one key, can't comp. sign the tx + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex'] + "00"]) + assert_raises_rpc_error(-22, "Missing transactions", self.nodes[0].combinerawtransaction, []) rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) self.log.debug(rawTxComb) self.nodes[2].sendrawtransaction(rawTxComb) @@ -606,6 +608,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + assert_raises_rpc_error(-25, "Input not found or already spent", self.nodes[0].combinerawtransaction, [rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) if __name__ == '__main__': diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 7edf9f3679..a357ae4d34 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -26,7 +26,7 @@ ServiceProxy class: - HTTP connections persist for the life of the AuthServiceProxy object (if server supports HTTP/1.1) -- sends protocol 'version', per JSON-RPC 1.1 +- sends "jsonrpc":"2.0", per JSON-RPC 2.0 - sends proper, incrementing 'id' - sends Basic HTTP authentication headers - parses all JSON numbers that look like floats as Decimal @@ -117,7 +117,7 @@ class AuthServiceProxy(): params = dict(args=args, **argsn) else: params = args or argsn - return {'version': '1.1', + return {'jsonrpc': '2.0', 'method': self._service_name, 'params': params, 'id': AuthServiceProxy.__id_count} @@ -125,15 +125,28 @@ class AuthServiceProxy(): def __call__(self, *args, **argsn): postdata = json.dumps(self.get_request(*args, **argsn), default=serialization_fallback, ensure_ascii=self.ensure_ascii) response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) - if response['error'] is not None: - raise JSONRPCException(response['error'], status) - elif 'result' not in response: - raise JSONRPCException({ - 'code': -343, 'message': 'missing JSON-RPC result'}, status) - elif status != HTTPStatus.OK: - raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + # For backwards compatibility tests, accept JSON RPC 1.1 responses + if 'jsonrpc' not in response: + if response['error'] is not None: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC result'}, status) + elif status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + else: + return response['result'] else: + assert response['jsonrpc'] == '2.0' + if status != HTTPStatus.OK: + raise JSONRPCException({ + 'code': -342, 'message': 'non-200 HTTP status code'}, status) + if 'error' in response: + raise JSONRPCException(response['error'], status) + elif 'result' not in response: + raise JSONRPCException({ + 'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status) return response['result'] def batch(self, rpc_call_list): @@ -142,7 +155,7 @@ class AuthServiceProxy(): response, status = self._request('POST', self.__url.path, postdata.encode('utf-8')) if status != HTTPStatus.OK: raise JSONRPCException({ - 'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status) + 'code': -342, 'message': 'non-200 HTTP status code'}, status) return response def _get_response(self): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 4231cbf495..9e44a11143 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2022 The Bitcoin Core developers +# Copyright (c) 2014-present The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -636,11 +636,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def find_conn(node, peer_subversion, inbound): return next(filter(lambda peer: peer['subver'] == peer_subversion and peer['inbound'] == inbound, node.getpeerinfo()), None) - # poll until version handshake complete to avoid race conditions - # with transaction relaying - # See comments in net_processing: - # * Must have a version message before anything else - # * Must have a verack message before anything else self.wait_until(lambda: find_conn(from_connection, to_connection_subver, inbound=False) is not None) self.wait_until(lambda: find_conn(to_connection, from_connection_subver, inbound=True) is not None) @@ -648,11 +643,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert peer is not None, "Error: peer disconnected" return peer['bytesrecv_per_msg'].pop(msg_type, 0) >= min_bytes_recv - self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'verack', 21)) - self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'verack', 21)) - - # The message bytes are counted before processing the message, so make - # sure it was fully processed by waiting for a ping. + # Poll until version handshake (fSuccessfullyConnected) is complete to + # avoid race conditions, because some message types are blocked from + # being sent or received before fSuccessfullyConnected. + # + # As the flag fSuccessfullyConnected is not exposed, check it by + # waiting for a pong, which can only happen after the flag was set. self.wait_until(lambda: check_bytesrecv(find_conn(from_connection, to_connection_subver, inbound=False), 'pong', 29)) self.wait_until(lambda: check_bytesrecv(find_conn(to_connection, from_connection_subver, inbound=True), 'pong', 29)) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 94ae96312e..0ac0af27d5 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -241,7 +241,7 @@ class TestNode(): if self.start_perf: self._start_perf() - def wait_for_rpc_connection(self): + def wait_for_rpc_connection(self, *, wait_for_import=True): """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect.""" # Poll at a rate of four times per second poll_per_s = 4 @@ -263,7 +263,7 @@ class TestNode(): ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up - if self.version_is_at_least(190000): + if self.version_is_at_least(190000) and wait_for_import: # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) self.wait_until(lambda: rpc.getmempoolinfo()['loaded']) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 113191d8a1..cb0d291361 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -7,6 +7,7 @@ from copy import deepcopy from decimal import Decimal from enum import Enum +import math from typing import ( Any, Optional, @@ -33,10 +34,13 @@ from test_framework.messages import ( CTxInWitness, CTxOut, hash256, + ser_compact_size, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( CScript, LEAF_VERSION_TAPSCRIPT, + OP_1, OP_NOP, OP_RETURN, OP_TRUE, @@ -52,6 +56,7 @@ from test_framework.script_util import ( from test_framework.util import ( assert_equal, assert_greater_than_or_equal, + get_fee, ) from test_framework.wallet_util import generate_keypair @@ -119,13 +124,16 @@ class MiniWallet: """Pad a transaction with extra outputs until it reaches a target weight (or higher). returns the tx """ - tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'a']))) + tx.vout.append(CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN]))) + # determine number of needed padding bytes by converting weight difference to vbytes dummy_vbytes = (target_weight - tx.get_weight() + 3) // 4 - tx.vout[-1].scriptPubKey = CScript([OP_RETURN, b'a' * dummy_vbytes]) - # Lower bound should always be off by at most 3 + # compensate for the increase of the compact-size encoded script length + # (note that the length encoding of the unpadded output script needs one byte) + dummy_vbytes -= len(ser_compact_size(dummy_vbytes)) - 1 + tx.vout[-1].scriptPubKey = CScript([OP_RETURN] + [OP_1] * dummy_vbytes) + # Actual weight should be at most 3 higher than target weight assert_greater_than_or_equal(tx.get_weight(), target_weight) - # Higher bound should always be off by at most 3 + 12 weight (for encoding the length) - assert_greater_than_or_equal(target_weight + 15, tx.get_weight()) + assert_greater_than_or_equal(target_weight + 3, tx.get_weight()) def get_balance(self): return sum(u['value'] for u in self._utxos) @@ -367,6 +375,10 @@ class MiniWallet: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False + if target_weight and not fee: # respect fee_rate if target weight is passed + # the actual weight might be off by 3 WUs, so calculate based on that (see self._bulk_tx) + max_actual_weight = target_weight + 3 + fee = get_fee(math.ceil(max_actual_weight / WITNESS_SCALE_FACTOR), fee_rate) send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) # create tx diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 725b116281..84e524558f 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -361,6 +361,7 @@ BASE_SCRIPTS = [ 'feature_addrman.py', 'feature_asmap.py', 'feature_fastprune.py', + 'feature_framework_miniwallet.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', 'mempool_accept_wtxid.py', diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 482667a26a..9818d73fdf 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -58,6 +58,7 @@ unsigned-integer-overflow:TxConfirmStats::EstimateMedianVal unsigned-integer-overflow:prevector.h unsigned-integer-overflow:EvalScript unsigned-integer-overflow:xoroshiro128plusplus.h +unsigned-integer-overflow:bitset_detail::PopCount implicit-integer-sign-change:CBlockPolicyEstimator::processBlockTx implicit-integer-sign-change:SetStdinEcho implicit-integer-sign-change:compressor.h |