diff options
190 files changed, 2059 insertions, 1423 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index eeb2d6590b..2f908dc718 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,8 +5,6 @@ configuration: Release platform: x64 clone_depth: 5 environment: - APPVEYOR_SAVE_CACHE_ON_ERROR: true - CLCACHE_SERVER: 1 PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' @@ -14,59 +12,41 @@ environment: QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -cache: -- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt -- C:\Qt5.9.8_x64_static_vs2019 install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: -# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file -# will cause the appveyor cache rules to invalidate the directory) -# 2. If the directory is missing: # a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, # b. Install the missing packages. - ps: | $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt - Write-Host "vcpkg list: $env:PACKAGES" - if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git pull origin master - git checkout $env:VCPKG_COMMIT_ID - .\bootstrap-vcpkg.bat - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null - cd "$env:APPVEYOR_BUILD_FOLDER" - } - else { - Write-Host "required vcpkg packages already installed." - } - c:\tools\vcpkg\vcpkg integrate install + Write-Host "vcpkg installing packages: $env:PACKAGES" + cd c:\tools\vcpkg + $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. + git pull origin master > $null + git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID + .\bootstrap-vcpkg.bat > $null + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" + .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + Write-Host "vcpkg packages installed successfully." + .\vcpkg integrate install + cd "$env:APPVEYOR_BUILD_FOLDER" before_build: # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# 1. If the Qt destination directory exists assume it is correct and do nothing. To -# force a fresh install of the packages delete the job's appveyor cache. -# 2. Otherwise: # a. Download the zip file with the prebuilt Qt static libraries. # b. Check that the downloaded file matches the expected hash. # c. Extract the zip file to the specific destination path expected by the msbuild projects. - ps: | - if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } + Write-Host "Downloading Qt binaries."; + Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; + Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; + if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { + Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; + Write-Host "Qt binary download matched the expected hash."; } else { - Write-Host "Qt binaries already present."; + Write-Host "ERROR: Qt binary download did not match the expected hash."; + Exit-AppveyorBuild; } - cmd: python build_msvc\msvc-autogen.py build_script: diff --git a/.cirrus.yml b/.cirrus.yml index 9c2598d606..446d3e35a9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,4 +1,5 @@ -# Global defaults +### Global defaults + timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits @@ -12,7 +13,9 @@ env: TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" -# Global task template + +### Global task template + # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks global_task_template: &GLOBAL_TASK_TEMPLATE ccache_cache: @@ -31,6 +34,7 @@ global_task_template: &GLOBAL_TASK_TEMPLATE - git merge FETCH_HEAD # Merge base to detect silent merge conflicts ci_script: - ./ci/test_run_all.sh + #task: # name: "Windows" # windows_container: @@ -48,6 +52,14 @@ global_task_template: &GLOBAL_TASK_TEMPLATE # - choco install python --version=3.7.7 -y task: + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:focal + env: + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + +task: name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' << : *GLOBAL_TASK_TEMPLATE container: diff --git a/.travis.yml b/.travis.yml index 9ab2227116..783737c8c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,11 +100,9 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' - # Not enough memory on travis machines, so feature_block is excluded for now + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: memory (MSan)]' env: >- - TEST_RUNNER_EXTRA="--exclude feature_block" - FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" + FILE_ENV="./ci/test/00_setup_env_native_msan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' @@ -117,6 +115,11 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" - stage: test + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" + + - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 35ba8425b3..fbbe1a2156 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -137,18 +137,6 @@ don't. */ #define HAVE_DECL_STRNLEN 1 -/* Define to 1 if you have the declaration of `__builtin_clz', and to 0 if you - don't. */ -//#define HAVE_DECL___BUILTIN_CLZ 1 - -/* Define to 1 if you have the declaration of `__builtin_clzl', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZL 1 - -/* Define to 1 if you have the declaration of `__builtin_clzll', and to 0 if - you don't. */ -//#define HAVE_DECL___BUILTIN_CLZLL 1 - /* Define to 1 if you have the <dlfcn.h> header file. */ /* #undef HAVE_DLFCN_H */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index c09997d39d..4fd516fff5 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -107,10 +107,10 @@ <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> - <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 26b576c1ae..fae424051d 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -6,11 +6,15 @@ export LC_ALL=C -travis_retry pip3 install codespell==1.15.0 -travis_retry pip3 install flake8==3.7.8 +travis_retry sudo apt update && sudo apt install -y clang-format-9 +sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 + +travis_retry pip3 install codespell==1.17.1 +travis_retry pip3 install flake8==3.8.3 travis_retry pip3 install yq -travis_retry pip3 install mypy==0.700 +travis_retry pip3 install mypy==0.781 -SHELLCHECK_VERSION=v0.6.0 -curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ +SHELLCHECK_VERSION=v0.7.1 +curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index a4dc54d1c1..45a29928cb 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,7 +9,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" -export OSX_SDK=10.14 +export XCODE_VERSION=11.3.1 +export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index f50efcc33a..eb0ac33184 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -11,7 +11,6 @@ export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to ma export PIP_PACKAGES="zmq" export GOAL="install" export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" -export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test, see https://github.com/bitcoin/bitcoin/pull/17240#issuecomment-546022121 why the other tests are disabled export RUN_SECURITY_TESTS="true" # Run without depends export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 225f77df92..d57c673069 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -10,5 +10,6 @@ export CONTAINER_NAME=ci_native_asan export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 +export TEST_RUNNER_EXTRA="--timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh new file mode 100644 index 0000000000..cc583edf17 --- /dev/null +++ b/ci/test/00_setup_env_native_msan.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:20.04" +LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/" +export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" +LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" +export BDB_PREFIX="${BASE_ROOT_DIR}/db4" + +export CONTAINER_NAME="ci_native_msan" +export PACKAGES="clang-9 llvm-9 cmake" +export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export USE_MEMORY_SANITIZER="true" +export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 553dab1491..1a0b14b62b 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md -export PACKAGES="python3-zmq" +export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 87b9d9da95..c4fe0eff2f 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,5 +10,6 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" +export TEST_RUNNER_EXTRA="--exclude feature_block --timeout-factor=4" # Increase timeout because sanitizers slow down. Low memory on Travis machines, exclude feature_block. export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index ff7c5fe913..0041122f1e 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind --timeout-factor=4" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 165983d906..7cbf6f9d10 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -90,6 +90,15 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" + DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" +fi + if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 3685504524..057395eba6 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -15,11 +15,22 @@ fi DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources -if [ -n "$OSX_SDK" ] && [ ! -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" +OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" + +if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" +fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB + # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for + # details. + DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" fi -if [ -n "$OSX_SDK" ] && [ -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - DOCKER_EXEC tar -C ${DEPENDS_DIR}/SDKs -xf ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + +if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b68cd9d3f8..17d765b862 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -37,6 +37,14 @@ END_FOLD set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # MemorySanitizer (MSAN) does not support tracking memory initialization done by + # using the Linux getrandom syscall. Avoid using getrandom by undefining + # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for + # details. + DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' +fi + BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD diff --git a/configure.ac b/configure.ac index 12bece6903..e1b09dae33 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,12 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = x; then + AC_MSG_ERROR([pkg-config not found]) +fi + BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli @@ -229,16 +235,6 @@ AC_ARG_ENABLE([zmq], [use_zmq=$enableval], [use_zmq=yes]) -AC_ARG_ENABLE([bip70], - [AS_HELP_STRING([--enable-bip70], - [BIP70 (payment protocol) support in the GUI (no longer supported)])], - [enable_bip70=$enableval], - [enable_bip70=no]) - -if test x$enable_bip70 != xno; then - AC_MSG_ERROR([BIP70 is no longer supported!]) -fi - AC_ARG_WITH([libmultiprocess], [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], @@ -570,13 +566,8 @@ AC_ARG_WITH([daemon], [build_bitcoind=$withval], [build_bitcoind=yes]) -use_pkgconfig=yes case $host in *mingw*) - - dnl pkgconfig does more harm than good with MinGW - use_pkgconfig=no - TARGET_OS=windows AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) @@ -635,9 +626,10 @@ case $host in bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) qt5_prefix=$($BREW --prefix qt5 2>/dev/null) - if test x$bdb_prefix != x; then - CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" - LIBS="$LIBS -L$bdb_prefix/lib" + if test x$bdb_prefix != x && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + dnl This must precede the call to BITCOIN_FIND_BDB48 below. + BDB_CFLAGS="-I$bdb_prefix/include" + BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi if test x$qt5_prefix != x; then PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" @@ -679,16 +671,6 @@ case $host in ;; esac -if test x$use_pkgconfig = xyes; then - m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) - m4_ifdef([PKG_PROG_PKG_CONFIG], [ - PKG_PROG_PKG_CONFIG - if test x"$PKG_CONFIG" = "x"; then - AC_MSG_ERROR(pkg-config not found.) - fi - ]) -fi - if test x$use_extended_functional_tests != xno; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi @@ -849,7 +831,21 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) -AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) +AC_MSG_CHECKING(for __builtin_clzl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzl(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT(no)] +) + +AC_MSG_CHECKING(for __builtin_clzll) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ + (void) __builtin_clzll(0); + ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT(no)] +) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) AC_MSG_CHECKING(for getmemoryinfo) @@ -1298,115 +1294,66 @@ CPPFLAGS="$TEMP_CPPFLAGS" fi -if test x$use_pkgconfig = xyes; then - : dnl - m4_ifdef( - [PKG_CHECK_MODULES], - [ - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) - fi - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) - if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) - fi - fi - - if test "x$use_zmq" = "xyes"; then - PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) - use_zmq=no]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi - ] - ) -else +dnl libevent check - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) - AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) - if test x$TARGET_OS != xwindows; then - AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) - fi +if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi +fi - if test "x$use_zmq" = "xyes"; then - AC_CHECK_HEADER([zmq.h], - [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], - [AC_MSG_WARN([zmq.h not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, - [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) - use_zmq=no - AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) - else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) - fi +dnl QR Code encoding library check - if test "x$use_zmq" = "xyes"; then - dnl Assume libzmq was built for static linking - case $host in - *mingw*) - ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" - ;; - esac - fi +if test "x$use_qr" != xno; then + BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) +fi - if test x$use_qr != xno; then - BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) - BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) - fi +dnl ZMQ check + +if test "x$use_zmq" = xyes; then + PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], + AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), + [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + use_zmq=no]) +else + AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) +fi + +if test "x$use_zmq" = xyes; then + dnl Assume libzmq was built for static linking + case $host in + *mingw*) + ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" + ;; + esac fi dnl univalue check need_bundled_univalue=yes - if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then need_bundled_univalue=no else - -if test x$system_univalue != xno ; then - found_univalue=no - if test x$use_pkgconfig = xyes; then - : #NOP - m4_ifdef( - [PKG_CHECK_MODULES], - [ - PKG_CHECK_MODULES([UNIVALUE],[libunivalue >= 1.0.4],[found_univalue=yes],[true]) - ] - ) - else - AC_CHECK_HEADER([univalue.h],[ - AC_CHECK_LIB([univalue], [main],[ - UNIVALUE_LIBS=-lunivalue - found_univalue=yes - ],[true]) - ],[true]) + if test x$system_univalue != xno; then + PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) + if test x$found_univalue = xyes; then + system_univalue=yes + need_bundled_univalue=no + elif test x$system_univalue = xyes; then + AC_MSG_ERROR([univalue not found]) + else + system_univalue=no + fi fi - if test x$found_univalue = xyes ; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes ; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no + if test x$need_bundled_univalue = xyes; then + UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' + UNIVALUE_LIBS='univalue/libunivalue.la' fi fi -if test x$need_bundled_univalue = xyes ; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' -fi - -fi - AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) @@ -1415,12 +1362,10 @@ dnl libmultiprocess library check libmultiprocess_found=no if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then - if test "x$use_pkgconfig" = xyes; then - m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ - libmultiprocess_found=yes; - libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; - ], [true])]) - fi + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) elif test "x$with_libmultiprocess" != xno; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 4a3df93cea..d498c9e2c8 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -209,7 +209,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.14.sdk.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index bbae7201e5..e0aaafc15a 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -32,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "MacOSX10.14.sdk.tar.gz" +- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail @@ -90,7 +90,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 68ebb5def1..fe677e3a1f 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,55 +14,44 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction -Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from -[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +### Step 1: Obtaining `Xcode.app` + +Our current macOS SDK +(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be +extracted from +[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). An Apple ID is needed to download this. -`Xcode.app` is packaged in a `.xip` archive. -This makes the SDK less-trivial to extract on non-macOS machines. -One approach (tested on Debian Buster) is outlined below: +After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` +archive. This makes the SDK less-trivial to extract on non-macOS machines. One +approach (tested on Debian Buster) is outlined below: ```bash +# Install/clone tools needed for extracting Xcode.app +apt install cpio +git clone https://github.com/bitcoin-core/apple-sdk-tools.git -apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make - -git clone https://github.com/tpoechtrager/xar -pushd xar/xar -./configure -make -make install -popd - -git clone https://github.com/NiklasRosenstein/pbzx -pushd pbzx -clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib -popd - -xar -xf Xcode_10.2.1.xip -C . - -./pbzx/pbzx -n Content | cpio -i - -find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; +# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current +# working directory +python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i ``` -on macOS the process is more straightforward: +On macOS the process is more straightforward: ```bash -xip -x Xcode_10.2.1.xip -tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +xip -x Xcode_11.3.1.xip ``` -Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from -[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). -The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First -ensure the DMG file is in the current directory, and then run the script. You -may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the -directory) when you've confirmed the extraction succeeded. +### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` + +To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run +the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the +previous stage) as the first argument. ```bash -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk +# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from +# the supplied Xcode.app +./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` ## Deterministic macOS DMG Notes @@ -91,13 +80,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: ```bash -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk ``` See the SDK Extraction notes above for how to obtain it. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk new file mode 100755 index 0000000000..457d8f5e64 --- /dev/null +++ b/contrib/macdeploy/gen-sdk @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse +import plistlib +import pathlib +import sys +import tarfile +import gzip +import os +import contextlib + +@contextlib.contextmanager +def cd(path): + """Context manager that restores PWD even if an exception was raised.""" + old_pwd = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(old_pwd) + +def run(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) + parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + + args = parser.parse_args() + + xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) + + xcode_app_plist = xcode_app.joinpath("Contents/version.plist") + with xcode_app_plist.open('rb') as fp: + pl = plistlib.load(fp) + xcode_version = pl['CFBundleShortVersionString'] + xcode_build_id = pl['ProductBuildVersion'] + print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)) + + sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk") + sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist") + with sdk_plist.open('rb') as fp: + pl = plistlib.load(fp) + sdk_version = pl['ProductVersion'] + sdk_build_id = pl['ProductBuildVersion'] + print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id)) + + out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) + + xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1") + assert xcode_libcxx_dir.is_dir() + + if args.out_sdktgz: + out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) + else: + # Construct our own out_sdktgz if not specified on the command line + out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + + def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): + """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files' + names + + e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking: + + tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir") + + would result in the following members being added to tarfp: + + foo/bar/ -> corresponding to /root/bazdir + foo/bar/qux -> corresponding to /root/bazdir/qux + + """ + def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.startswith("./"): + tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) + if tarinfo.linkname and tarinfo.linkname.startswith("./"): + tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + return tarinfo + with cd(dir_to_add): + tarfp.add(".", recursive=True, filter=change_tarinfo_base) + + print("Creating output .tar.gz file...") + with out_sdktgz_path.open("wb") as fp: + with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with tarfile.open(mode="w", fileobj=gzf) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Adding libc++ headers...") + tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name)) + print("Done! Find the resulting gzipped tarball at:") + print(out_sdktgz_path.resolve()) + +if __name__ == '__main__': + run() diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index 573a71a675..eaca473b40 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -4,5 +4,5 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_keys_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_keys_invalid.json + PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 50 > ../../src/test/data/key_io_valid.json + PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 50 > ../../src/test/data/key_io_invalid.json diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 82e086a326..0d4fab937d 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,15 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.14 -OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ +OSX_SDK_VERSION=10.15.1 +XCODE_VERSION=11.3.1 +XCODE_BUILD_ID=11C505 +LD64_VERSION=530 + +OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers + +# When cross-compiling for Darwin using Clang, -mlinker-version must be passed to +# ensure that modern linker features are enabled. +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index eb45e14f6f..1cd5a1749a 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,17 +3,23 @@ $(package)_version=2.1.11-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=229393ab2bf0dc94694f21836846b424f3532585bac3468738b7bf752c03901e +$(package)_patches=0001-fix-windows-getaddrinfo.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/0001-fix-windows-getaddrinfo.patch && \ ./autogen.sh endef +# When building for Windows, we set _WIN32_WINNT to target the same Windows +# version as we do in configure. Due to quirks in libevents build system, this +# is also required to enable support for ipv6. See #19375. define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic + $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef define $(package)_config_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 4195230b40..bdebd11862 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,14 +1,14 @@ package=native_cctools -$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 +$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec $(package)_build_subdir=cctools -$(package)_clang_version=6.0.1 +$(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 +$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive @@ -72,7 +72,5 @@ define $(package)_stage_cmds cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \ - cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi + cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil endef diff --git a/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch new file mode 100644 index 0000000000..a98cd90bd5 --- /dev/null +++ b/depends/patches/libevent/0001-fix-windows-getaddrinfo.patch @@ -0,0 +1,15 @@ +diff -ur libevent-2.1.8-stable.orig/configure.ac libevent-2.1.8-stable/configure.ac +--- libevent-2.1.8-stable.orig/configure.ac 2017-01-29 17:51:00.000000000 +0000 ++++ libevent-2.1.8-stable/configure.ac 2020-03-07 01:11:16.311335005 +0000 +@@ -389,6 +389,10 @@ + #ifdef HAVE_NETDB_H + #include <netdb.h> + #endif ++#ifdef _WIN32 ++#include <winsock2.h> ++#include <ws2tcpip.h> ++#endif + ]], + [[ + getaddrinfo; +Only in libevent-2.1.8-stable: configure.ac~ diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 11e9c90f49..842a3964df 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -61,7 +61,6 @@ Only supports JSON as output format. * pruned : (boolean) if the blocks are subject to pruning * pruneheight : (numeric) highest block available * softforks : (array) status of softforks in progress -* bip9_softforks : (object) status of BIP9 softforks in progress #### Query UTXO set `GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>` @@ -79,7 +78,6 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 "bitmap": "1", "utxos" : [ { - "txvers" : 1 "height" : 2147483647, "value" : 8.8687, "scriptPubKey" : { diff --git a/doc/build-windows.md b/doc/build-windows.md index d3dc467f19..28b6aceb3c 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -91,15 +91,22 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. +Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause +unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach +is to temporarily disable WSL support for Win32 applications. + Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var + sudo bash -c "echo 0 > /proc/sys/fs/binfmt_misc/status" # Disable WSL support for Win32 applications. cd depends make HOST=x86_64-w64-mingw32 cd .. ./autogen.sh # not required when building from tarball CONFIG_SITE=$PWD/depends/x86_64-w64-mingw32/share/config.site ./configure --prefix=/ make + sudo bash -c "echo 1 > /proc/sys/fs/binfmt_misc/status" # Enable WSL support for Win32 applications. ## Depends system diff --git a/doc/developer-notes.md b/doc/developer-notes.md index b33b3ad18a..bd3daa3202 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -620,6 +620,19 @@ class A - *Rationale*: Easier to understand what is happening, thus easier to spot mistakes, even for those that are not language lawyers. +- Use `Span` as function argument when it can operate on any range-like container. + + - *Rationale*: Compared to `Foo(const vector<int>&)` this avoids the need for a (potentially expensive) + conversion to vector if the caller happens to have the input stored in another type of container. + However, be aware of the pitfalls documented in [span.h](../src/span.h). + +```cpp +void Foo(Span<const int> data); + +std::vector<int> vec{1,2,3}; +Foo(vec); +``` + - Prefer `enum class` (scoped enumerations) over `enum` (traditional enumerations) where possible. - *Rationale*: Scoped enumerations avoid two potential pitfalls/problems with traditional C++ enumerations: implicit conversions to `int`, and name clashes due to enumerators being exported to the surrounding scope. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 419b1db44e..c97b8d4d50 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -121,6 +121,8 @@ $ git clone https://github.com/google/afl $ make -C afl/ $ make -C afl/llvm_mode/ $ ./autogen.sh +# It is possible to compile with afl-gcc and afl-g++ instead of afl-clang. However, running afl-fuzz +# may require more memory via the -m flag. $ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz $ make # For macOS you may need to ignore x86 compilation checks when running "make". If so, diff --git a/doc/release-notes-11413.md b/doc/release-notes-11413.md new file mode 100644 index 0000000000..32735e37f6 --- /dev/null +++ b/doc/release-notes-11413.md @@ -0,0 +1,11 @@ +Updated or changed RPC +---------------------- + +The `bumpfee`, `fundrawtransaction`, `sendmany`, `sendtoaddress`, and `walletcreatefundedpsbt` +RPC commands have been updated to include two new fee estimation methods "BTC/kB" and "sat/B". +The target is the fee expressed explicitly in the given form. Note that use of this feature +will trigger BIP 125 (replace-by-fee) opt-in. + +In addition, the `estimate_mode` parameter is now case insensitive for all of the above RPC commands. + +The `bumpfee` command now uses `conf_target` rather than `confTarget` in the options. diff --git a/doc/release-notes-18594.md b/doc/release-notes-18594.md new file mode 100644 index 0000000000..6a2ef0a67c --- /dev/null +++ b/doc/release-notes-18594.md @@ -0,0 +1,5 @@ +## CLI + +The `bitcoin-cli -getinfo` command now displays the wallet name and balance for +each of the loaded wallets when more than one is loaded (e.g. in multiwallet +mode) and a wallet is not specified with `-rpcwallet`. (#18594) diff --git a/doc/release-notes-19133.md b/doc/release-notes-19133.md new file mode 100644 index 0000000000..5150fbe1c7 --- /dev/null +++ b/doc/release-notes-19133.md @@ -0,0 +1,7 @@ +## CLI + +A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` +followed by `generatetoaddress`, can generate blocks for command line testing +purposes. This is a client-side version of the +[former](https://github.com/bitcoin/bitcoin/issues/14299) `generate` RPC. See +the help for details. (#19133) diff --git a/doc/release-notes-19219.md b/doc/release-notes-19219.md new file mode 100644 index 0000000000..b5ee885ddc --- /dev/null +++ b/doc/release-notes-19219.md @@ -0,0 +1,23 @@ +#### Changes regarding misbehaving peers + +Peers that misbehave (e.g. send us invalid blocks) are now referred to as +discouraged nodes in log output, as they're not (and weren't) strictly banned: +incoming connections are still allowed from them, but they're preferred for +eviction. + +Furthermore, a few additional changes are introduced to how discouraged +addresses are treated: + +- Discouraging an address does not time out automatically after 24 hours + (or the `-bantime` setting). Depending on traffic from other peers, + discouragement may time out at an indeterminate time. + +- Discouragement is not persisted over restarts. + +- There is no method to list discouraged addresses. They are not returned by + the `listbanned` RPC. That RPC also no longer reports the `ban_reason` + field, as `"manually added"` is the only remaining option. + +- Discouragement cannot be removed with the `setban remove` RPC command. + If you need to remove a discouragement, you can remove all discouragements by + stop-starting your node. diff --git a/doc/release-notes.md b/doc/release-notes.md index d9d0ecd631..e73bedfb10 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -45,6 +45,11 @@ wallet versions of Bitcoin Core are generally supported. Compatibility ============== +During this release cycle, work has been done to ensure that the codebase is fully +compatible with C++17. The intention is to begin using C++17 features starting +with the 0.22.0 release. This means that a compiler that supports C++17 will be +required to compile 0.22.0. + Bitcoin Core is supported and extensively tested on operating systems using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as diff --git a/src/.clang-format b/src/.clang-format index a8f8565f80..ef7a0ef5c7 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -3,7 +3,6 @@ AccessModifierOffset: -4 AlignAfterOpenBracket: true AlignEscapedNewlinesLeft: true AlignTrailingComments: true -AllowAllArgumentsOnNextLine : true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: true diff --git a/src/Makefile.am b/src/Makefile.am index 632ed3e31f..cd3cc95707 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,9 +110,9 @@ BITCOIN_CORE_H = \ banman.h \ base58.h \ bech32.h \ - bloom.h \ blockencodings.h \ blockfilter.h \ + bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -133,6 +133,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + dbwrapper.h \ flatfile.h \ fs.h \ httprpc.h \ @@ -148,7 +149,6 @@ BITCOIN_CORE_H = \ interfaces/wallet.h \ key.h \ key_io.h \ - dbwrapper.h \ limitedmap.h \ logging.h \ logging/timer.h \ @@ -167,6 +167,7 @@ BITCOIN_CORE_H = \ node/context.h \ node/psbt.h \ node/transaction.h \ + node/ui_interface.h \ node/utxo_snapshot.h \ noui.h \ optional.h \ @@ -206,13 +207,12 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadsafety.h \ threadinterrupt.h \ + threadsafety.h \ timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ - ui_interface.h \ undo.h \ util/asmap.h \ util/bip32.h \ @@ -221,8 +221,6 @@ BITCOIN_CORE_H = \ util/error.h \ util/fees.h \ util/golombrice.h \ - util/spanparsing.h \ - util/system.h \ util/macros.h \ util/memory.h \ util/message.h \ @@ -230,19 +228,22 @@ BITCOIN_CORE_H = \ util/rbf.h \ util/ref.h \ util/settings.h \ + util/spanparsing.h \ util/string.h \ + util/system.h \ util/threadnames.h \ util/time.h \ util/translation.h \ + util/ui_change_type.h \ util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ versionbits.h \ versionbitsinfo.h \ - walletinitinterface.h \ wallet/bdb.h \ wallet/coincontrol.h \ + wallet/coinselection.h \ wallet/context.h \ wallet/crypter.h \ wallet/db.h \ @@ -257,7 +258,7 @@ BITCOIN_CORE_H = \ wallet/walletdb.h \ wallet/wallettool.h \ wallet/walletutil.h \ - wallet/coinselection.h \ + walletinitinterface.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -286,16 +287,16 @@ libbitcoin_server_a_SOURCES = \ blockfilter.cpp \ chain.cpp \ consensus/tx_verify.cpp \ + dbwrapper.cpp \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ index/txindex.cpp \ + init.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ - init.cpp \ - dbwrapper.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ @@ -304,6 +305,7 @@ libbitcoin_server_a_SOURCES = \ node/context.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/ui_interface.cpp \ noui.cpp \ policy/fees.cpp \ policy/rbf.cpp \ @@ -322,7 +324,6 @@ libbitcoin_server_a_SOURCES = \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ - ui_interface.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 1f66516172..e5c19e5afc 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -276,8 +276,6 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_IMAGES = - RES_MOVIES = $(wildcard $(srcdir)/qt/res/movies/spinner-*.png) BITCOIN_RC = qt/res/bitcoin-qt-res.rc @@ -290,7 +288,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_MOVIES) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -361,7 +359,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_MOVIES) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 03cd9133c8..9dc3078487 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -32,6 +32,7 @@ FUZZ_TARGETS = \ test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ test/fuzz/coins_view \ + test/fuzz/crypto \ test/fuzz/crypto_common \ test/fuzz/cuckoocache \ test/fuzz/decode_tx \ @@ -231,6 +232,7 @@ BITCOIN_TESTS =\ test/net_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ + test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ @@ -479,6 +481,12 @@ test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_view_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp +test_fuzz_crypto_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_crypto_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_crypto_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_crypto_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_SOURCES = test/fuzz/crypto.cpp + test_fuzz_crypto_common_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_common_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_common_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/addrdb.h b/src/addrdb.h index c6d4307d69..8410c3776c 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -17,13 +17,6 @@ class CSubNet; class CAddrMan; class CDataStream; -typedef enum BanReason -{ - BanReasonUnknown = 0, - BanReasonNodeMisbehaving = 1, - BanReasonManuallyAdded = 2 -} BanReason; - class CBanEntry { public: @@ -31,7 +24,6 @@ public: int nVersion; int64_t nCreateTime; int64_t nBanUntil; - uint8_t banReason; CBanEntry() { @@ -44,31 +36,17 @@ public: nCreateTime = nCreateTimeIn; } - explicit CBanEntry(int64_t n_create_time_in, BanReason ban_reason_in) : CBanEntry(n_create_time_in) + SERIALIZE_METHODS(CBanEntry, obj) { - banReason = ban_reason_in; + uint8_t ban_reason = 2; //! For backward compatibility + READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); } - SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } - void SetNull() { nVersion = CBanEntry::CURRENT_VERSION; nCreateTime = 0; nBanUntil = 0; - banReason = BanReasonUnknown; - } - - std::string banReasonToString() const - { - switch (banReason) { - case BanReasonNodeMisbehaving: - return "node misbehaving"; - case BanReasonManuallyAdded: - return "manually added"; - default: - return "unknown"; - } } }; diff --git a/src/banman.cpp b/src/banman.cpp index 9cc584f0e4..8752185a60 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -6,7 +6,7 @@ #include <banman.h> #include <netaddress.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/time.h> #include <util/translation.h> @@ -26,7 +26,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node ips/subnets from banlist.dat %dms\n", - banmap.size(), GetTimeMillis() - n_start); + m_banned.size(), GetTimeMillis() - n_start); } else { LogPrintf("Invalid or missing banlist.dat; recreating\n"); SetBannedSetDirty(true); // force write @@ -68,28 +68,13 @@ void BanMan::ClearBanned() if (m_client_interface) m_client_interface->BannedListChanged(); } -int BanMan::IsBannedLevel(CNetAddr net_addr) +bool BanMan::IsDiscouraged(const CNetAddr& net_addr) { - // Returns the most severe level of banning that applies to this address. - // 0 - Not banned - // 1 - Automatic misbehavior ban - // 2 - Any other ban - int level = 0; - auto current_time = GetTime(); LOCK(m_cs_banned); - for (const auto& it : m_banned) { - CSubNet sub_net = it.first; - CBanEntry ban_entry = it.second; - - if (current_time < ban_entry.nBanUntil && sub_net.Match(net_addr)) { - if (ban_entry.banReason != BanReasonNodeMisbehaving) return 2; - level = 1; - } - } - return level; + return m_discouraged.contains(net_addr.GetAddrBytes()); } -bool BanMan::IsBanned(CNetAddr net_addr) +bool BanMan::IsBanned(const CNetAddr& net_addr) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -104,7 +89,7 @@ bool BanMan::IsBanned(CNetAddr net_addr) return false; } -bool BanMan::IsBanned(CSubNet sub_net) +bool BanMan::IsBanned(const CSubNet& sub_net) { auto current_time = GetTime(); LOCK(m_cs_banned); @@ -118,15 +103,21 @@ bool BanMan::IsBanned(CSubNet sub_net) return false; } -void BanMan::Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CNetAddr& net_addr, int64_t ban_time_offset, bool since_unix_epoch) { CSubNet sub_net(net_addr); - Ban(sub_net, ban_reason, ban_time_offset, since_unix_epoch); + Ban(sub_net, ban_time_offset, since_unix_epoch); +} + +void BanMan::Discourage(const CNetAddr& net_addr) +{ + LOCK(m_cs_banned); + m_discouraged.insert(net_addr.GetAddrBytes()); } -void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset, bool since_unix_epoch) +void BanMan::Ban(const CSubNet& sub_net, int64_t ban_time_offset, bool since_unix_epoch) { - CBanEntry ban_entry(GetTime(), ban_reason); + CBanEntry ban_entry(GetTime()); int64_t normalized_ban_time_offset = ban_time_offset; bool normalized_since_unix_epoch = since_unix_epoch; @@ -146,8 +137,8 @@ void BanMan::Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ba } if (m_client_interface) m_client_interface->BannedListChanged(); - //store banlist to disk immediately if user requested ban - if (ban_reason == BanReasonManuallyAdded) DumpBanlist(); + //store banlist to disk immediately + DumpBanlist(); } bool BanMan::Unban(const CNetAddr& net_addr) diff --git a/src/banman.h b/src/banman.h index 6bea2e75e9..f6bfbd1e49 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,6 +6,7 @@ #define BITCOIN_BANMAN_H #include <addrdb.h> +#include <bloom.h> #include <fs.h> #include <net_types.h> // For banmap_t #include <sync.h> @@ -23,32 +24,55 @@ class CClientUIInterface; class CNetAddr; class CSubNet; -// Denial-of-service detection/prevention -// The idea is to detect peers that are behaving -// badly and disconnect/ban them, but do it in a -// one-coding-mistake-won't-shatter-the-entire-network -// way. -// IMPORTANT: There should be nothing I can give a -// node that it will forward on that will make that -// node's peers drop it. If there is, an attacker -// can isolate a node and/or try to split the network. -// Dropping a node for sending stuff that is invalid -// now but might be valid in a later version is also -// dangerous, because it can cause a network split -// between nodes running old code and nodes running -// new code. +// Banman manages two related but distinct concepts: +// +// 1. Banning. This is configured manually by the user, through the setban RPC. +// If an address or subnet is banned, we never accept incoming connections from +// it and never create outgoing connections to it. We won't gossip its address +// to other peers in addr messages. Banned addresses and subnets are stored to +// banlist.dat on shutdown and reloaded on startup. Banning can be used to +// prevent connections with spy nodes or other griefers. +// +// 2. Discouragement. If a peer misbehaves enough (see Misbehaving() in +// net_processing.cpp), we'll mark that address as discouraged. We still allow +// incoming connections from them, but they're preferred for eviction when +// we receive new incoming connections. We never make outgoing connections to +// them, and do not gossip their address to other peers. This is implemented as +// a bloom filter. We can (probabilistically) test for membership, but can't +// list all discouraged addresses or unmark them as discouraged. Discouragement +// can prevent our limited connection slots being used up by incompatible +// or broken peers. +// +// Neither banning nor discouragement are protections against denial-of-service +// attacks, since if an attacker has a way to waste our resources and we +// disconnect from them and ban that address, it's trivial for them to +// reconnect from another IP address. +// +// Attempting to automatically disconnect or ban any class of peer carries the +// risk of splitting the network. For example, if we banned/disconnected for a +// transaction that fails a policy check and a future version changes the +// policy check so the transaction is accepted, then that transaction could +// cause the network to split between old nodes and new nodes. class BanMan { public: ~BanMan(); BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time); - void Ban(const CNetAddr& net_addr, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Ban(const CSubNet& sub_net, const BanReason& ban_reason, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false); + void Discourage(const CNetAddr& net_addr); void ClearBanned(); - int IsBannedLevel(CNetAddr net_addr); - bool IsBanned(CNetAddr net_addr); - bool IsBanned(CSubNet sub_net); + + //! Return whether net_addr is banned + bool IsBanned(const CNetAddr& net_addr); + + //! Return whether sub_net is exactly banned + bool IsBanned(const CSubNet& sub_net); + + //! Return whether net_addr is discouraged. + bool IsDiscouraged(const CNetAddr& net_addr); + bool Unban(const CNetAddr& net_addr); bool Unban(const CSubNet& sub_net); void GetBanned(banmap_t& banmap); @@ -68,6 +92,7 @@ private: CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; + CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; #endif diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index b8e8717896..3dcce92ab5 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -13,9 +13,9 @@ #include <init.h> #include <interfaces/chain.h> #include <node/context.h> +#include <node/ui_interface.h> #include <noui.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> diff --git a/src/bloom.cpp b/src/bloom.cpp index 54fcf487e4..d182f0728e 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -135,8 +135,8 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY) { std::vector<std::vector<unsigned char> > vSolutions; - txnouttype type = Solver(txout.scriptPubKey, vSolutions); - if (type == TX_PUBKEY || type == TX_MULTISIG) { + TxoutType type = Solver(txout.scriptPubKey, vSolutions); + if (type == TxoutType::PUBKEY || type == TxoutType::MULTISIG) { insert(COutPoint(hash, i)); } } diff --git a/src/core_write.cpp b/src/core_write.cpp index eb0cc35f06..69b62df901 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -131,20 +131,20 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) { CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); ssTx << tx; - return HexStr(ssTx.begin(), ssTx.end()); + return HexStr(ssTx); } void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) { out.pushKV("asm", ScriptToAsmStr(script)); - out.pushKV("hex", HexStr(script.begin(), script.end())); + out.pushKV("hex", HexStr(script)); std::vector<std::vector<unsigned char>> solns; - txnouttype type = Solver(script, solns); + TxoutType type = Solver(script, solns); out.pushKV("type", GetTxnOutputType(type)); CTxDestination address; - if (include_address && ExtractDestination(script, address) && type != TX_PUBKEY) { + if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } } @@ -152,15 +152,15 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) { - txnouttype type; + TxoutType type; std::vector<CTxDestination> addresses; int nRequired; out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); if (fIncludeHex) - out.pushKV("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + out.pushKV("hex", HexStr(scriptPubKey)); - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TX_PUBKEY) { + if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TxoutType::PUBKEY) { out.pushKV("type", GetTxnOutputType(type)); return; } @@ -190,19 +190,19 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, const CTxIn& txin = tx.vin[i]; UniValue in(UniValue::VOBJ); if (tx.IsCoinBase()) - in.pushKV("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + in.pushKV("coinbase", HexStr(txin.scriptSig)); else { in.pushKV("txid", txin.prevout.hash.GetHex()); in.pushKV("vout", (int64_t)txin.prevout.n); UniValue o(UniValue::VOBJ); o.pushKV("asm", ScriptToAsmStr(txin.scriptSig, true)); - o.pushKV("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + o.pushKV("hex", HexStr(txin.scriptSig)); in.pushKV("scriptSig", o); } if (!tx.vin[i].scriptWitness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : tx.vin[i].scriptWitness.stack) { - txinwitness.push_back(HexStr(item.begin(), item.end())); + txinwitness.push_back(HexStr(item)); } in.pushKV("txinwitness", txinwitness); } diff --git a/src/crypto/common.h b/src/crypto/common.h index e7bb020a19..5b4932c992 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -82,12 +82,12 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) /** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ uint64_t static inline CountBits(uint64_t x) { -#if HAVE_DECL___BUILTIN_CLZL +#if HAVE_BUILTIN_CLZL if (sizeof(unsigned long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; } #endif -#if HAVE_DECL___BUILTIN_CLZLL +#if HAVE_BUILTIN_CLZLL if (sizeof(unsigned long long) >= sizeof(uint64_t)) { return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 116d7d8679..215b033708 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -292,18 +292,6 @@ public: // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; - // not available for LevelDB; provide for compatibility with BDB - bool Flush() - { - return true; - } - - bool Sync() - { - CDBBatch batch(*this); - return WriteBatch(batch, true); - } - CDBIterator *NewIterator() { return new CDBIterator(*this, pdb->NewIterator(iteroptions)); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5e78fd1d71..1e5ea2de83 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -7,10 +7,10 @@ #include <chainparamsbase.h> #include <compat.h> #include <netbase.h> +#include <node/ui_interface.h> #include <rpc/protocol.h> // For HTTP status codes #include <shutdown.h> #include <sync.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> diff --git a/src/index/base.cpp b/src/index/base.cpp index a93b67395d..f587205a28 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,9 +4,9 @@ #include <chainparams.h> #include <index/base.h> +#include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 59d1888fff..64472714cc 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <index/txindex.h> +#include <node/ui_interface.h> #include <shutdown.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> diff --git a/src/init.cpp b/src/init.cpp index 8d9566edc3..2e39bcd9a5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -31,6 +31,7 @@ #include <net_processing.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -48,8 +49,8 @@ #include <torcontrol.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/asmap.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> @@ -430,8 +431,8 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-bantime=<n>", strprintf("Number of seconds to keep misbehaving peers from reconnecting (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting and discouraging misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-bind=<addr>", "Bind to given address and always listen on it. Use [host]:port notation for IPv6", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1317,8 +1318,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - CScheduler::Function serviceLoop = [&node]{ node.scheduler->serviceQueue(); }; - threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1379,9 +1379,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) node.mempool = &::mempool; assert(!node.chainman); node.chainman = &g_chainman; - ChainstateManager& chainman = EnsureChainman(node); + ChainstateManager& chainman = *Assert(node.chainman); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.chainman, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1589,7 +1589,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). - if (!::BlockIndex().empty() && + if (!chainman.BlockIndex().empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } @@ -1869,8 +1869,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) //// debug print { LOCK(cs_main); - LogPrintf("block tree size = %u\n", ::BlockIndex().size()); - chain_active_height = ::ChainActive().Height(); + LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); + chain_active_height = chainman.ActiveChain().Height(); } LogPrintf("nBestHeight = %d\n", chain_active_height); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index d1e04b114d..d49e4454af 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -13,6 +13,7 @@ #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -25,7 +26,6 @@ #include <sync.h> #include <timedata.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <univalue.h> #include <util/system.h> diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index d420788dbe..834a16ecf5 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -17,6 +17,7 @@ #include <netaddress.h> #include <netbase.h> #include <node/context.h> +#include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/settings.h> @@ -26,7 +27,6 @@ #include <support/allocators/secure.h> #include <sync.h> #include <txmempool.h> -#include <ui_interface.h> #include <util/ref.h> #include <util/system.h> #include <util/translation.h> @@ -146,10 +146,10 @@ public: } return false; } - bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) override + bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override { if (m_context.banman) { - m_context.banman->Ban(net_addr, reason, ban_time_offset); + m_context.banman->Ban(net_addr, ban_time_offset); return true; } return false; diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 877a40568f..b88b5bc14e 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -122,7 +122,7 @@ public: virtual bool getBanned(banmap_t& banmap) = 0; //! Ban node. - virtual bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) = 0; + virtual bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) = 0; //! Unban node. virtual bool unban(const CSubNet& ip) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index b65eb72b1c..f6806aed65 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -13,11 +13,11 @@ #include <script/standard.h> #include <support/allocators/secure.h> #include <sync.h> -#include <ui_interface.h> #include <uint256.h> #include <util/check.h> #include <util/ref.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -438,7 +438,6 @@ public: bool canGetAddresses() override { return m_wallet->CanGetAddresses(); } bool privateKeysDisabled() override { return m_wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); } OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } - OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index e2161521f6..3cdadbc72e 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -9,8 +9,8 @@ #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination #include <support/allocators/secure.h> // For SecureString -#include <ui_interface.h> // For ChangeType #include <util/message.h> +#include <util/ui_change_type.h> #include <functional> #include <map> @@ -256,9 +256,6 @@ public: // Get default address type. virtual OutputType getDefaultAddressType() = 0; - // Get default change type. - virtual OutputType getDefaultChangeType() = 0; - //! Get max tx fee. virtual CAmount getDefaultMaxTxFee() = 0; diff --git a/src/net.cpp b/src/net.cpp index afde8a894b..0cafb57e80 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -14,12 +14,12 @@ #include <clientversion.h> #include <consensus/consensus.h> #include <crypto/sha256.h> -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> +#include <node/ui_interface.h> #include <protocol.h> #include <random.h> #include <scheduler.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/translation.h> @@ -1011,17 +1011,24 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // on all platforms. Set it again here just to be sure. SetSocketNoDelay(hSocket); - int bannedlevel = m_banman ? m_banman->IsBannedLevel(addr) : 0; - - // Don't accept connections from banned peers, but if our inbound slots aren't almost full, accept - // if the only banning reason was an automatic misbehavior ban. - if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && bannedlevel > ((nInbound + 1 < nMaxInbound) ? 1 : 0)) + // Don't accept connections from banned peers. + bool banned = m_banman->IsBanned(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && banned) { LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); CloseSocket(hSocket); return; } + // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. + bool discouraged = m_banman->IsDiscouraged(addr); + if (!NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_NOBAN) && nInbound + 1 >= nMaxInbound && discouraged) + { + LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + if (nInbound >= nMaxInbound) { if (!AttemptToEvictConnection()) { @@ -1045,7 +1052,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { pnode->m_permissionFlags = permissionFlags; // If this flag is present, the user probably expect that RPC and QT report it as whitelisted (backward compatibility) pnode->m_legacyWhitelisted = legacyWhitelisted; - pnode->m_prefer_evict = bannedlevel > 0; + pnode->m_prefer_evict = discouraged; m_msgproc->InitializeNode(pnode); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -1104,12 +1111,9 @@ void CConnman::DisconnectNodes() if (pnode->GetRefCount() <= 0) { bool fDelete = false; { - TRY_LOCK(pnode->cs_inventory, lockInv); - if (lockInv) { - TRY_LOCK(pnode->cs_vSend, lockSend); - if (lockSend) { - fDelete = true; - } + TRY_LOCK(pnode->cs_vSend, lockSend); + if (lockSend) { + fDelete = true; } } if (fDelete) { @@ -2046,10 +2050,10 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai return; } if (!pszDest) { - if (IsLocal(addrConnect) || - FindNode(static_cast<CNetAddr>(addrConnect)) || (m_banman && m_banman->IsBanned(addrConnect)) || - FindNode(addrConnect.ToStringIPPort())) + bool banned_or_discouraged = m_banman && (m_banman->IsDiscouraged(addrConnect) || m_banman->IsBanned(addrConnect)); + if (IsLocal(addrConnect) || FindNode(static_cast<CNetAddr>(addrConnect)) || banned_or_discouraged || FindNode(addrConnect.ToStringIPPort())) { return; + } } else if (FindNode(std::string(pszDest))) return; @@ -803,7 +803,7 @@ public: // There is no final sorting before sending, as they are always sent immediately // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - RecursiveMutex cs_inventory; + Mutex cs_inventory; struct TxRelay { mutable RecursiveMutex cs_filter; @@ -982,18 +982,6 @@ public: } } - void PushBlockInventory(const uint256& hash) - { - LOCK(cs_inventory); - vInventoryBlockToSend.push_back(hash); - } - - void PushBlockHash(const uint256 &hash) - { - LOCK(cs_inventory); - vBlockHashesToAnnounce.push_back(hash); - } - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap); diff --git a/src/net_permissions.h b/src/net_permissions.h index e004067e75..2cf85120fe 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -24,7 +24,7 @@ enum NetPermissionFlags // Always relay transactions from this peer, even if already in mempool // Keep parameter interaction: forcerelay implies relay PF_FORCERELAY = (1U << 2) | PF_RELAY, - // Can't be banned for misbehavior + // Can't be banned/disconnected/discouraged for misbehavior PF_NOBAN = (1U << 4), // Can query the mempool PF_MEMPOOL = (1U << 5), diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 270e415e42..f1a89a8936 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -13,10 +13,9 @@ #include <consensus/validation.h> #include <hash.h> #include <index/blockfilterindex.h> -#include <validation.h> #include <merkleblock.h> -#include <netmessagemaker.h> #include <netbase.h> +#include <netmessagemaker.h> #include <policy/fees.h> #include <policy/policy.h> #include <primitives/block.h> @@ -26,16 +25,14 @@ #include <scheduler.h> #include <tinyformat.h> #include <txmempool.h> -#include <util/system.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> +#include <util/system.h> +#include <validation.h> #include <memory> #include <typeinfo> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - /** Expiration time for orphan transactions in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ @@ -189,7 +186,7 @@ namespace { * We use this to avoid requesting transactions that have already been * confirnmed. */ - RecursiveMutex g_cs_recent_confirmed_transactions; + Mutex g_cs_recent_confirmed_transactions; std::unique_ptr<CRollingBloomFilter> g_recent_confirmed_transactions GUARDED_BY(g_cs_recent_confirmed_transactions); /** Blocks that are in flight, and that are in the queue to be downloaded. */ @@ -252,8 +249,8 @@ struct CNodeState { bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. int nMisbehavior; - //! Whether this peer should be disconnected and banned (unless whitelisted). - bool fShouldBan; + //! Whether this peer should be disconnected and marked as discouraged (unless whitelisted with noban). + bool m_should_discourage; //! String name of this peer (debugging/logging purposes). const std::string name; //! The best known block we know this peer has announced. @@ -404,7 +401,7 @@ struct CNodeState { { fCurrentlyConnected = false; nMisbehavior = 0; - fShouldBan = false; + m_should_discourage = false; pindexBestKnownBlock = nullptr; hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = nullptr; @@ -1019,7 +1016,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) } /** - * Mark a misbehaving peer to be banned depending upon the value of `-banscore`. + * Increment peer's misbehavior score. If the new value surpasses banscore (specified on startup or by default), mark node to be discouraged, meaning the peer might be disconnected & added to the discouragement filter. */ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { @@ -1035,14 +1032,14 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIV std::string message_prefixed = message.empty() ? "" : (": " + message); if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) { - LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) BAN THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); - state->fShouldBan = true; + LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); + state->m_should_discourage = true; } else LogPrint(BCLog::NET, "%s: %s peer=%d (%d -> %d)%s\n", __func__, state->name, pnode, state->nMisbehavior-howmuch, state->nMisbehavior, message_prefixed); } /** - * Potentially ban a node based on the contents of a BlockValidationState object + * Potentially mark a node discouraged based on the contents of a BlockValidationState object * * @param[in] via_compact_block this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact @@ -1072,7 +1069,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s break; } - // Ban outbound (but not inbound) peers if on an invalid chain. + // Discourage outbound (but not inbound) peers if on an invalid chain. // Exempt HB compact block peers and manual connections. if (!via_compact_block && !node_state->m_is_inbound && !node_state->m_is_manual_connection) { Misbehaving(nodeid, 100, message); @@ -1107,7 +1104,7 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s } /** - * Potentially ban a node based on the contents of a TxValidationState object + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object * * @return Returns true if the peer was punished (probably disconnected) */ @@ -1328,9 +1325,10 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } // Relay inventory, but don't relay old inventory during initial block download. connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + LOCK(pnode->cs_inventory); if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { for (const uint256& hash : reverse_iterate(vHashes)) { - pnode->PushBlockHash(hash); + pnode->vBlockHashesToAnnounce.push_back(hash); } } }); @@ -1339,7 +1337,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } /** - * Handle invalid block rejection and consequent peer banning, maintain which + * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { @@ -1607,7 +1605,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom.hashContinue) { - // Bypass PushBlockInventory, this must send even if redundant, + // Send immediately. This must send even if redundant, // and we want it right after the last block so they don't // wait for other stuff first. std::vector<CInv> vInv; @@ -2454,7 +2452,7 @@ void ProcessMessage( if (vAddr.size() > 1000) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message addr size() = %u", vAddr.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("addr message size = %u", vAddr.size())); return; } @@ -2476,7 +2474,8 @@ void ProcessMessage( if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom.AddAddressKnown(addr); - if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them + if (banman->IsDiscouraged(addr)) continue; // Do not process banned/discouraged addresses beyond remembering we received them + if (banman->IsBanned(addr)) continue; bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -2530,7 +2529,7 @@ void ProcessMessage( if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message inv size() = %u", vInv.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("inv message size = %u", vInv.size())); return; } @@ -2576,7 +2575,7 @@ void ProcessMessage( LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; return; - } else if (!fAlreadyHave && !fImporting && !fReindex && !::ChainstateActive().IsInitialBlockDownload()) { + } else if (!fAlreadyHave && !chainman.ActiveChainstate().IsInitialBlockDownload()) { RequestTx(State(pfrom.GetId()), inv.hash, current_time); } } @@ -2596,7 +2595,7 @@ void ProcessMessage( if (vInv.size() > MAX_INV_SZ) { LOCK(cs_main); - Misbehaving(pfrom.GetId(), 20, strprintf("message getdata size() = %u", vInv.size())); + Misbehaving(pfrom.GetId(), 20, strprintf("getdata message size = %u", vInv.size())); return; } @@ -2666,7 +2665,7 @@ void ProcessMessage( LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } - pfrom.PushBlockInventory(pindex->GetBlockHash()); + WITH_LOCK(pfrom.cs_inventory, pfrom.vInventoryBlockToSend.push_back(pindex->GetBlockHash())); if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll @@ -3118,7 +3117,7 @@ void ProcessMessage( // relayed before full validation (see BIP 152), so we don't want to disconnect // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that - // will be detected and the peer will be banned. + // will be detected and the peer will be disconnected/discouraged. return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } @@ -3204,7 +3203,7 @@ void ProcessMessage( // 3. the block is otherwise invalid (eg invalid coinbase, // block is too big, too many legacy sigops, etc). // So if CheckBlock failed, #3 is the only possibility. - // Under BIP 152, we don't DoS-ban unless proof of work is + // Under BIP 152, we don't discourage the peer unless proof of work is // invalid (we don't require all the stateless checks to have // been run). This is handled below, so just treat this as // though the block was successfully read, and rely on the @@ -3329,7 +3328,7 @@ void ProcessMessage( std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { - if (!banman->IsBanned(addr)) { + if (!banman->IsDiscouraged(addr) && !banman->IsBanned(addr)) { pfrom.PushAddress(addr, insecure_rand); } } @@ -3564,25 +3563,26 @@ void ProcessMessage( return; } -bool PeerLogicValidation::CheckIfBanned(CNode& pnode) +bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) { AssertLockHeld(cs_main); CNodeState &state = *State(pnode.GetId()); - if (state.fShouldBan) { - state.fShouldBan = false; - if (pnode.HasPermission(PF_NOBAN)) + if (state.m_should_discourage) { + state.m_should_discourage = false; + if (pnode.HasPermission(PF_NOBAN)) { LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode.addr.ToString()); - else if (pnode.m_manual_connection) + } else if (pnode.m_manual_connection) { LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode.addr.ToString()); - else if (pnode.addr.IsLocal()) { - // Disconnect but don't ban _this_ local node - LogPrintf("Warning: disconnecting but not banning local peer %s!\n", pnode.addr.ToString()); + } else if (pnode.addr.IsLocal()) { + // Disconnect but don't discourage this local node + LogPrintf("Warning: disconnecting but not discouraging local peer %s!\n", pnode.addr.ToString()); pnode.fDisconnect = true; } else { - // Disconnect and ban all nodes sharing the address + // Disconnect and discourage all nodes sharing the address + LogPrintf("Disconnecting and discouraging peer %s!\n", pnode.addr.ToString()); if (m_banman) { - m_banman->Ban(pnode.addr, BanReasonNodeMisbehaving); + m_banman->Discourage(pnode.addr); } connman->DisconnectNode(pnode.addr); } @@ -3682,7 +3682,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } LOCK(cs_main); - CheckIfBanned(*pfrom); + MaybeDiscourageAndDisconnect(*pfrom); return fMoreWork; } @@ -3885,7 +3885,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!lockMain) return true; - if (CheckIfBanned(*pto)) return true; + if (MaybeDiscourageAndDisconnect(*pto)) return true; CNodeState &state = *State(pto->GetId()); @@ -4083,7 +4083,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { - pto->PushBlockInventory(hashToAnnounce); + pto->vInventoryBlockToSend.push_back(hashToAnnounce); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, pto->GetId(), hashToAnnounce.ToString()); } @@ -4392,10 +4392,21 @@ bool PeerLogicValidation::SendMessages(CNode* pto) ) { CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); int64_t timeNow = GetTimeMicros(); + static FeeFilterRounder g_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}}; + if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + // Received tx-inv messages are discarded when the active + // chainstate is in IBD, so tell the peer to not send them. + currentFilter = MAX_MONEY; + } else { + static const CAmount MAX_FILTER{g_filter_rounder.round(MAX_MONEY)}; + if (pto->m_tx_relay->lastSentFeeFilter == MAX_FILTER) { + // Send the current filter if we sent MAX_FILTER previously + // and made it out of IBD. + pto->m_tx_relay->nextSendTimeFeeFilter = timeNow - 1; + } + } if (timeNow > pto->m_tx_relay->nextSendTimeFeeFilter) { - static CFeeRate default_feerate(DEFAULT_MIN_RELAY_TX_FEE); - static FeeFilterRounder filterRounder(default_feerate); - CAmount filterToSend = filterRounder.round(currentFilter); + CAmount filterToSend = g_filter_rounder.round(currentFilter); // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); if (filterToSend != pto->m_tx_relay->lastSentFeeFilter) { diff --git a/src/net_processing.h b/src/net_processing.h index 19beca0cc4..eadf29e59f 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -31,7 +31,7 @@ private: ChainstateManager& m_chainman; CTxMemPool& m_mempool; - bool CheckIfBanned(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool MaybeDiscourageAndDisconnect(CNode& pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); public: PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); diff --git a/src/netaddress.h b/src/netaddress.h index d7ab537cf6..c201012154 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -90,6 +90,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector<bool> &asmap) const; std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; + std::vector<unsigned char> GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index e3c4c828b6..fb46ea1731 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,13 +8,23 @@ #include <coins.h> #include <hash.h> #include <serialize.h> -#include <validation.h> #include <uint256.h> #include <util/system.h> +#include <validation.h> #include <map> -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static uint64_t GetBogoSize(const CScript& scriptPubKey) +{ + return 32 /* txid */ + + 4 /* vout index */ + + 4 /* height + coinbase */ + + 8 /* amount */ + + 2 /* scriptPubKey len */ + + scriptPubKey.size() /* scriptPubKey */; +} + +static void ApplyStats(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); ss << hash; @@ -26,26 +36,38 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); } ss << VARINT(0u); } +static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + stats.nTransactions++; + for (const auto& output : outputs) { + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += GetBogoSize(output.second.out.scriptPubKey); + } +} + //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point) +template <typename T> +static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point) { stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } - ss << stats.hashBlock; + + PrepareHash(hash_obj, stats); + uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { @@ -54,7 +76,7 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -66,9 +88,38 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); + ApplyStats(stats, hash_obj, prevkey, outputs); } - stats.hashSerialized = ss.GetHash(); + + FinalizeHash(hash_obj, stats); + stats.nDiskSize = view->EstimateSize(); return true; } + +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point) +{ + switch (hash_type) { + case(CoinStatsHashType::HASH_SERIALIZED): { + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + return GetUTXOStats(view, stats, ss, interruption_point); + } + case(CoinStatsHashType::NONE): { + return GetUTXOStats(view, stats, nullptr, interruption_point); + } + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +// The legacy hash serializes the hashBlock +static void PrepareHash(CHashWriter& ss, CCoinsStats& stats) +{ + ss << stats.hashBlock; +} +static void PrepareHash(std::nullptr_t, CCoinsStats& stats) {} + +static void FinalizeHash(CHashWriter& ss, CCoinsStats& stats) +{ + stats.hashSerialized = ss.GetHash(); +} +static void FinalizeHash(std::nullptr_t, CCoinsStats& stats) {} diff --git a/src/node/coinstats.h b/src/node/coinstats.h index d9cdaa3036..2a7441c10e 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -14,6 +14,11 @@ class CCoinsView; +enum class CoinStatsHashType { + HASH_SERIALIZED, + NONE, +}; + struct CCoinsStats { int nHeight{0}; @@ -30,6 +35,6 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index c45d9e6689..c783c39cd6 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -49,10 +49,4 @@ struct NodeContext { ~NodeContext(); }; -inline ChainstateManager& EnsureChainman(const NodeContext& node) -{ - assert(node.chainman); - return *node.chainman; -} - #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/ui_interface.cpp b/src/node/ui_interface.cpp index 15795bd67f..8d3665975d 100644 --- a/src/ui_interface.cpp +++ b/src/node/ui_interface.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> -#include <boost/signals2/last_value.hpp> +#include <boost/signals2/optional_last_value.hpp> #include <boost/signals2/signal.hpp> CClientUIInterface uiInterface; struct UISignals { - boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::last_value<bool>> ThreadSafeMessageBox; - boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::last_value<bool>> ThreadSafeQuestion; + boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::optional_last_value<bool>> ThreadSafeMessageBox; + boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::optional_last_value<bool>> ThreadSafeQuestion; boost::signals2::signal<CClientUIInterface::InitMessageSig> InitMessage; boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; @@ -42,8 +42,8 @@ ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); -bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } -bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } +bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style).value_or(false);} +bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style).value_or(false);} void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } diff --git a/src/ui_interface.h b/src/node/ui_interface.h index b7895e373f..d574ab879f 100644 --- a/src/ui_interface.h +++ b/src/node/ui_interface.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UI_INTERFACE_H -#define BITCOIN_UI_INTERFACE_H +#ifndef BITCOIN_NODE_UI_INTERFACE_H +#define BITCOIN_NODE_UI_INTERFACE_H #include <functional> #include <memory> @@ -20,14 +20,6 @@ class connection; } } // namespace boost -/** General change type (added, updated, removed). */ -enum ChangeType -{ - CT_NEW, - CT_UPDATED, - CT_DELETED -}; - /** Signals for UI communication. */ class CClientUIInterface { @@ -122,8 +114,8 @@ void InitWarning(const bilingual_str& str); /** Show error message **/ bool InitError(const bilingual_str& str); -inline bool AbortError(const bilingual_str& str) { return InitError(str); } +constexpr auto AbortError = InitError; extern CClientUIInterface uiInterface; -#endif // BITCOIN_UI_INTERFACE_H +#endif // BITCOIN_NODE_UI_INTERFACE_H diff --git a/src/noui.cpp b/src/noui.cpp index 821d10e3bc..3c82512fac 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -6,7 +6,7 @@ #include <noui.h> #include <logging.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/translation.h> #include <string> diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 871474d56e..e978852826 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -42,8 +42,8 @@ const std::string& FormatOutputType(OutputType type) case OutputType::LEGACY: return OUTPUT_TYPE_STRING_LEGACY; case OutputType::P2SH_SEGWIT: return OUTPUT_TYPE_STRING_P2SH_SEGWIT; case OutputType::BECH32: return OUTPUT_TYPE_STRING_BECH32; - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) @@ -61,8 +61,8 @@ CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) return witdest; } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) @@ -100,6 +100,6 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, return ScriptHash(witprog); } } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(false); } diff --git a/src/outputtype.h b/src/outputtype.h index 1438f65844..77a16b1d05 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -18,14 +18,6 @@ enum class OutputType { LEGACY, P2SH_SEGWIT, BECH32, - - /** - * Special output type for change outputs only. Automatically choose type - * based on address type setting and the types other of non-change outputs - * (see -changetype option documentation and implementation in - * CWallet::TransactionChangeType for details). - */ - CHANGE_AUTO, }; extern const std::array<OutputType, 3> OUTPUT_TYPES; diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index 14be6192fe..a01e259731 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -7,8 +7,6 @@ #include <tinyformat.h> -const std::string CURRENCY_UNIT = "BTC"; - CFeeRate::CFeeRate(const CAmount& nFeePaid, size_t nBytes_) { assert(nBytes_ <= uint64_t(std::numeric_limits<int64_t>::max())); @@ -37,7 +35,10 @@ CAmount CFeeRate::GetFee(size_t nBytes_) const return nFee; } -std::string CFeeRate::ToString() const +std::string CFeeRate::ToString(const FeeEstimateMode& fee_estimate_mode) const { - return strprintf("%d.%08d %s/kB", nSatoshisPerK / COIN, nSatoshisPerK % COIN, CURRENCY_UNIT); + switch (fee_estimate_mode) { + case FeeEstimateMode::SAT_B: return strprintf("%d.%03d %s/B", nSatoshisPerK / 1000, nSatoshisPerK % 1000, CURRENCY_ATOM); + default: return strprintf("%d.%08d %s/kB", nSatoshisPerK / COIN, nSatoshisPerK % COIN, CURRENCY_UNIT); + } } diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 61fa80c130..883940f73c 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -11,7 +11,17 @@ #include <string> -extern const std::string CURRENCY_UNIT; +const std::string CURRENCY_UNIT = "BTC"; // One formatted unit +const std::string CURRENCY_ATOM = "sat"; // One indivisible minimum value unit + +/* Used to determine type of fee estimation requested */ +enum class FeeEstimateMode { + UNSET, //!< Use default settings based on other criteria + ECONOMICAL, //!< Force estimateSmartFee to use non-conservative estimates + CONSERVATIVE, //!< Force estimateSmartFee to use conservative estimates + BTC_KB, //!< Use explicit BTC/kB fee given in coin control + SAT_B, //!< Use explicit sat/B fee given in coin control +}; /** * Fee rate in satoshis per kilobyte: CAmount / kB @@ -46,7 +56,7 @@ public: friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } friend bool operator!=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK != b.nSatoshisPerK; } CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; + std::string ToString(const FeeEstimateMode& fee_estimate_mode = FeeEstimateMode::BTC_KB) const; SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; diff --git a/src/policy/fees.h b/src/policy/fees.h index 6ee6e0d547..e445c1590d 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -45,13 +45,6 @@ enum class FeeReason { REQUIRED, }; -/* Used to determine type of fee estimation requested */ -enum class FeeEstimateMode { - UNSET, //!< Use default settings based on other criteria - ECONOMICAL, //!< Force estimateSmartFee to use non-conservative estimates - CONSERVATIVE, //!< Force estimateSmartFee to use conservative estimates -}; - /* Used to return detailed information about a feerate bucket */ struct EstimatorBucket { diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 07d51c0088..c56abaf6c9 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -50,14 +50,14 @@ bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); } -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) +bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType) { std::vector<std::vector<unsigned char> > vSolutions; whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD) { + if (whichType == TxoutType::NONSTANDARD) { return false; - } else if (whichType == TX_MULTISIG) { + } else if (whichType == TxoutType::MULTISIG) { unsigned char m = vSolutions.front()[0]; unsigned char n = vSolutions.back()[0]; // Support up to x-of-3 multisig txns as standard @@ -65,7 +65,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) return false; if (m < 1 || m > n) return false; - } else if (whichType == TX_NULL_DATA && + } else if (whichType == TxoutType::NULL_DATA && (!fAcceptDatacarrier || scriptPubKey.size() > nMaxDatacarrierBytes)) { return false; } @@ -110,16 +110,16 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR } unsigned int nDataOut = 0; - txnouttype whichType; + TxoutType whichType; for (const CTxOut& txout : tx.vout) { if (!::IsStandard(txout.scriptPubKey, whichType)) { reason = "scriptpubkey"; return false; } - if (whichType == TX_NULL_DATA) + if (whichType == TxoutType::NULL_DATA) nDataOut++; - else if ((whichType == TX_MULTISIG) && (!permit_bare_multisig)) { + else if ((whichType == TxoutType::MULTISIG) && (!permit_bare_multisig)) { reason = "bare-multisig"; return false; } else if (IsDust(txout, dust_relay_fee)) { @@ -163,10 +163,10 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; - txnouttype whichType = Solver(prev.scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD) { + TxoutType whichType = Solver(prev.scriptPubKey, vSolutions); + if (whichType == TxoutType::NONSTANDARD) { return false; - } else if (whichType == TX_SCRIPTHASH) { + } else if (whichType == TxoutType::SCRIPTHASH) { std::vector<std::vector<unsigned char> > stack; // convert the scriptSig into a stack, so we can inspect the redeemScript if (!EvalScript(stack, tx.vin[i].scriptSig, SCRIPT_VERIFY_NONE, BaseSignatureChecker(), SigVersion::BASE)) diff --git a/src/policy/policy.h b/src/policy/policy.h index 1561a41c5e..7f168ee20f 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -81,7 +81,7 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); +bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType); /** * Check for standard transaction types * @return True if all outputs (scriptPubKeys) use only standard transaction forms diff --git a/src/psbt.cpp b/src/psbt.cpp index 10260740f0..3fb743e5db 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -35,14 +35,6 @@ bool PartiallySignedTransaction::Merge(const PartiallySignedTransaction& psbt) return true; } -bool PartiallySignedTransaction::IsSane() const -{ - for (PSBTInput input : inputs) { - if (!input.IsSane()) return false; - } - return true; -} - bool PartiallySignedTransaction::AddInput(const CTxIn& txin, PSBTInput& psbtin) { if (std::find(tx->vin.begin(), tx->vin.end(), txin) != tx->vin.end()) { @@ -144,8 +136,8 @@ void PSBTInput::Merge(const PSBTInput& input) { if (!non_witness_utxo && input.non_witness_utxo) non_witness_utxo = input.non_witness_utxo; if (witness_utxo.IsNull() && !input.witness_utxo.IsNull()) { + // TODO: For segwit v1, we will want to clear out the non-witness utxo when setting a witness one. For v0 and non-segwit, this is not safe witness_utxo = input.witness_utxo; - non_witness_utxo = nullptr; // Clear out any non-witness utxo when we set a witness one. } partial_sigs.insert(input.partial_sigs.begin(), input.partial_sigs.end()); @@ -158,18 +150,6 @@ void PSBTInput::Merge(const PSBTInput& input) if (final_script_witness.IsNull() && !input.final_script_witness.IsNull()) final_script_witness = input.final_script_witness; } -bool PSBTInput::IsSane() const -{ - // Cannot have both witness and non-witness utxos - if (!witness_utxo.IsNull() && non_witness_utxo) return false; - - // If we have a witness_script or a scriptWitness, we must also have a witness utxo - if (!witness_script.empty() && witness_utxo.IsNull()) return false; - if (!final_script_witness.IsNull() && witness_utxo.IsNull()) return false; - - return true; -} - void PSBTOutput::FillSignatureData(SignatureData& sigdata) const { if (!redeem_script.empty()) { @@ -261,11 +241,6 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& bool require_witness_sig = false; CTxOut utxo; - // Verify input sanity, which checks that at most one of witness or non-witness utxos is provided. - if (!input.IsSane()) { - return false; - } - if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; @@ -299,10 +274,11 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (require_witness_sig && !sigdata.witness) return false; input.FromSignatureData(sigdata); - // If we have a witness signature, use the smaller witness UTXO. + // If we have a witness signature, put a witness UTXO. + // TODO: For segwit v1, we should remove the non_witness_utxo if (sigdata.witness) { input.witness_utxo = utxo; - input.non_witness_utxo = nullptr; + // input.non_witness_utxo = nullptr; } // Fill in the missing info @@ -356,10 +332,6 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector return TransactionError::PSBT_MISMATCH; } } - if (!out.IsSane()) { - return TransactionError::INVALID_PSBT; - } - return TransactionError::OK; } diff --git a/src/psbt.h b/src/psbt.h index 0a8ea2ea0b..0951b76f83 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -62,18 +62,17 @@ struct PSBTInput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTInput& input); - bool IsSane() const; PSBTInput() {} template <typename Stream> inline void Serialize(Stream& s) const { // Write the utxo - // If there is a non-witness utxo, then don't add the witness one. if (non_witness_utxo) { SerializeToVector(s, PSBT_IN_NON_WITNESS_UTXO); OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); SerializeToVector(os, non_witness_utxo); - } else if (!witness_utxo.IsNull()) { + } + if (!witness_utxo.IsNull()) { SerializeToVector(s, PSBT_IN_WITNESS_UTXO); SerializeToVector(s, witness_utxo); } @@ -284,7 +283,6 @@ struct PSBTOutput void FillSignatureData(SignatureData& sigdata) const; void FromSignatureData(const SignatureData& sigdata); void Merge(const PSBTOutput& output); - bool IsSane() const; PSBTOutput() {} template <typename Stream> @@ -401,7 +399,6 @@ struct PartiallySignedTransaction /** Merge psbt into this. The two psbts must have the same underlying CTransaction (i.e. the * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ NODISCARD bool Merge(const PartiallySignedTransaction& psbt); - bool IsSane() const; bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} @@ -551,10 +548,6 @@ struct PartiallySignedTransaction if (outputs.size() != tx->vout.size()) { throw std::ios_base::failure("Outputs provided does not match the number of outputs in transaction."); } - // Sanity check - if (!IsSane()) { - throw std::ios_base::failure("PSBT is not sane."); - } } template <typename Stream> diff --git a/src/pubkey.h b/src/pubkey.h index 261842b7f7..4c28af4a4d 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -142,6 +142,9 @@ public: unsigned int len = ::ReadCompactSize(s); if (len <= SIZE) { s.read((char*)vch, len); + if (len != size()) { + Invalidate(); + } } else { // invalid pubkey, skip available data char dummy; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index e0b9345a32..c7dd16d2ed 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,11 +30,10 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <noui.h> -#include <ui_interface.h> #include <uint256.h> #include <util/system.h> -#include <util/translation.h> #include <util/threadnames.h> +#include <util/translation.h> #include <validation.h> #include <memory> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index b4182e8524..65f226a925 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -34,7 +34,7 @@ #include <chainparams.h> #include <interfaces/handler.h> #include <interfaces/node.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> @@ -683,6 +683,7 @@ void BitcoinGUI::removeWallet(WalletModel* walletModel) m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { setWalletActionsEnabled(false); + overviewAction->setChecked(true); } else if (m_wallet_selector->count() == 1) { m_wallet_selector_label_action->setVisible(false); m_wallet_selector_action->setVisible(false); diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 8b70800838..1217ca3e2e 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1078,12 +1078,6 @@ <height>426</height> </rect> </property> - <property name="minimumSize"> - <size> - <width>300</width> - <height>0</height> - </size> - </property> <layout class="QGridLayout" name="gridLayout_2" columnstretch="0,1"> <item row="0" column="0"> <widget class="QLabel" name="label_30"> diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index beca78a021..a1da85bda7 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -14,9 +14,9 @@ #include <chainparams.h> #include <interfaces/node.h> -#include <policy/policy.h> #include <key_io.h> -#include <ui_interface.h> +#include <node/ui_interface.h> +#include <policy/policy.h> #include <util/system.h> #include <wallet/wallet.h> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index dafd517ca8..71094f7112 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -467,6 +467,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty // Install event filter for up and down arrow ui->lineEdit->installEventFilter(this); + ui->lineEdit->setMaxLength(16 * 1024 * 1024); ui->messagesWidget->installEventFilter(this); connect(ui->clearButton, &QPushButton::clicked, this, &RPCConsole::clear); @@ -1218,7 +1219,7 @@ void RPCConsole::banSelectedNode(int bantime) // Find possible nodes, ban it and clear the selected node const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); if (stats) { - m_node.ban(stats->nodeStats.addr, BanReasonManuallyAdded, bantime); + m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); } } diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 0ac61f3adc..97fb88d71c 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,9 +21,9 @@ #include <chainparams.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <policy/fees.h> #include <txmempool.h> -#include <ui_interface.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> #include <wallet/wallet.h> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index ced6a299d5..6e6b2b8466 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -14,7 +14,6 @@ #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 52007ef350..632a18de5c 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -234,6 +234,7 @@ void TransactionRecord::updateStatus(const interfaces::WalletTxStatus& wtx, cons bool TransactionRecord::statusUpdateNeeded(const uint256& block_hash) const { + assert(!block_hash.IsNull()); return status.m_cur_block_hash != block_hash || status.needsUpdate; } diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 327a489488..c560dc58e7 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -178,21 +178,16 @@ public: TransactionRecord* index(interfaces::Wallet& wallet, const uint256& cur_block_hash, const int idx) { - if(idx >= 0 && idx < cachedWallet.size()) - { + if (idx >= 0 && idx < cachedWallet.size()) { TransactionRecord *rec = &cachedWallet[idx]; - // Get required locks upfront. This avoids the GUI from getting - // stuck if the core is holding the locks for a longer time - for - // example, during a wallet rescan. - // // If a status update is needed (blocks came in since last check), - // update the status of this transaction from the wallet. Otherwise, - // simply re-use the cached status. + // try to update the status of this transaction from the wallet. + // Otherwise, simply re-use the cached status. interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (rec->statusUpdateNeeded(cur_block_hash) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { + if (!cur_block_hash.IsNull() && rec->statusUpdateNeeded(cur_block_hash) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, cur_block_hash, numBlocks, block_time); } return rec; @@ -664,7 +659,7 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord* data = priv->index(walletModel->wallet(), walletModel->clientModel().getBestBlockHash(), row); + TransactionRecord* data = priv->index(walletModel->wallet(), walletModel->getLastBlockProcessed(), row); if(data) { return createIndex(row, column, data); diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 3df81807f0..54ecfc38ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -17,7 +17,7 @@ #include <qt/transactiontablemodel.h> #include <qt/walletmodel.h> -#include <ui_interface.h> +#include <node/ui_interface.h> #include <QApplication> #include <QComboBox> diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 72c75f7be0..e374dd191c 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -21,8 +21,8 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <node/ui_interface.h> #include <psbt.h> -#include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <util/translation.h> #include <wallet/coincontrol.h> @@ -87,7 +87,7 @@ void WalletModel::pollBalanceChanged() { // Avoid recomputing wallet balances unless a TransactionChanged or // BlockTip notification was received. - if (!fForceCheckBalanceChanged && m_cached_last_update_tip == m_client_model->getBestBlockHash()) return; + if (!fForceCheckBalanceChanged && m_cached_last_update_tip == getLastBlockProcessed()) return; // Try to get balances and return early if locks can't be acquired. This // avoids the GUI from getting stuck on periodical polls if the core is @@ -588,3 +588,8 @@ void WalletModel::refresh(bool pk_hash_only) { addressTableModel = new AddressTableModel(this, pk_hash_only); } + +uint256 WalletModel::getLastBlockProcessed() const +{ + return m_client_model ? m_client_model->getBestBlockHash() : uint256{}; +} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 38e8a14556..fd52db2da3 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -155,6 +155,9 @@ public: AddressTableModel* getAddressTableModel() const { return addressTableModel; } void refresh(bool pk_hash_only = false); + + uint256 getLastBlockProcessed() const; + private: std::unique_ptr<interfaces::Wallet> m_wallet; std::unique_ptr<interfaces::Handler> m_handler_unload; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index a3f303aea6..2fc883a5f5 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -4,9 +4,6 @@ #include <qt/walletview.h> -#include <node/psbt.h> -#include <node/transaction.h> -#include <policy/policy.h> #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> #include <qt/clientmodel.h> @@ -23,7 +20,8 @@ #include <qt/walletmodel.h> #include <interfaces/node.h> -#include <ui_interface.h> +#include <node/ui_interface.h> +#include <psbt.h> #include <util/strencodings.h> #include <QAction> diff --git a/src/rest.cpp b/src/rest.cpp index cde8b472d3..8cb594a03b 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -188,7 +188,7 @@ static bool rest_headers(const util::Ref& context, ssHeader << pindex->GetBlockHeader(); } - std::string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; + std::string strHex = HexStr(ssHeader) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -253,7 +253,7 @@ static bool rest_block(HTTPRequest* req, case RetFormat::HEX: { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; + std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -391,7 +391,7 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssTx << tx; - std::string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; + std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); return true; @@ -556,7 +556,7 @@ static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std: case RetFormat::HEX: { CDataStream ssGetUTXOResponse(SER_NETWORK, PROTOCOL_VERSION); ssGetUTXOResponse << ::ChainActive().Height() << ::ChainActive().Tip()->GetBlockHash() << bitmap << outs; - std::string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; + std::string strHex = HexStr(ssGetUTXOResponse) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8252af3ee6..98d4b2ce81 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -75,7 +75,10 @@ CTxMemPool& EnsureMemPool(const util::Ref& context) ChainstateManager& EnsureChainman(const util::Ref& context) { NodeContext& node = EnsureNodeContext(context); - return EnsureChainman(node); + if (!node.chainman) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node chainman not found"); + } + return *node.chainman; } /* Calculate the difficulty for a given block index. @@ -779,7 +782,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); ssBlock << pblockindex->GetBlockHeader(); - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -905,7 +908,7 @@ static UniValue getblock(const JSONRPCRequest& request) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + std::string strHex = HexStr(ssBlock); return strHex; } @@ -972,7 +975,9 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n", - {}, + { + {"hash_type", RPCArg::Type::STR, /* default */ "hash_serialized_2", "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'none'."}, + }, RPCResult{ RPCResult::Type::OBJ, "", "", { @@ -981,7 +986,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, - {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash"}, + {RPCResult::Type::STR_HEX, "hash_serialized_2", "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount"}, }}, @@ -996,14 +1001,18 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; ::ChainstateActive().ForceFlushStateToDisk(); + const CoinStatsHashType hash_type = ParseHashType(request.params[0], CoinStatsHashType::HASH_SERIALIZED); + CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats, RpcInterruptionPoint)) { + if (GetUTXOStats(coins_view, stats, hash_type, RpcInterruptionPoint)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + } ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { @@ -1325,50 +1334,48 @@ static UniValue getchaintips(const JSONRPCRequest& request) }, }.Check(request); + ChainstateManager& chainman = EnsureChainman(request.context); LOCK(cs_main); /* - * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: The set of chain tips is the active chain tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. - * - add ::ChainActive().Tip() + * - Add the active chain tip */ std::set<const CBlockIndex*, CompareBlocksByHeight> setTips; std::set<const CBlockIndex*> setOrphans; std::set<const CBlockIndex*> setPrevs; - for (const std::pair<const uint256, CBlockIndex*>& item : ::BlockIndex()) - { - if (!::ChainActive().Contains(item.second)) { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { + if (!chainman.ActiveChain().Contains(item.second)) { setOrphans.insert(item.second); setPrevs.insert(item.second->pprev); } } - for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) - { + for (std::set<const CBlockIndex*>::iterator it = setOrphans.begin(); it != setOrphans.end(); ++it) { if (setPrevs.erase(*it) == 0) { setTips.insert(*it); } } // Always report the currently active tip. - setTips.insert(::ChainActive().Tip()); + setTips.insert(chainman.ActiveChain().Tip()); /* Construct the output array. */ UniValue res(UniValue::VARR); - for (const CBlockIndex* block : setTips) - { + for (const CBlockIndex* block : setTips) { UniValue obj(UniValue::VOBJ); obj.pushKV("height", block->nHeight); obj.pushKV("hash", block->phashBlock->GetHex()); - const int branchLen = block->nHeight - ::ChainActive().FindFork(block)->nHeight; + const int branchLen = block->nHeight - chainman.ActiveChain().FindFork(block)->nHeight; obj.pushKV("branchlen", branchLen); std::string status; - if (::ChainActive().Contains(block)) { + if (chainman.ActiveChain().Contains(block)) { // This block is part of the currently active chain. status = "active"; } else if (block->nStatus & BLOCK_FAILED_MASK) { @@ -2159,7 +2166,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) UniValue unspent(UniValue::VOBJ); unspent.pushKV("txid", outpoint.hash.GetHex()); unspent.pushKV("vout", (int32_t)outpoint.n); - unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey.begin(), txo.scriptPubKey.end())); + unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); unspent.pushKV("height", (int32_t)coin.nHeight); @@ -2316,7 +2323,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, RpcInterruptionPoint)) { + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, RpcInterruptionPoint)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2377,7 +2384,7 @@ static const CRPCCommand commands[] = { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, { "blockchain", "getrawmempool", &getrawmempool, {"verbose"} }, { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {} }, + { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, { "blockchain", "savemempool", &savemempool, {} }, { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index e8e0ab3179..fee6a893eb 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -873,7 +873,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); if (!pblocktemplate->vchCoinbaseCommitment.empty()) { - result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment.begin(), pblocktemplate->vchCoinbaseCommitment.end())); + result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); } return result; diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index ce98a7c937..53d38f4e11 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -63,7 +63,7 @@ static UniValue validateaddress(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); @@ -131,7 +131,7 @@ static UniValue createmultisig(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); result.pushKV("descriptor", descriptor->ToString()); return result; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index df1e0fe623..393442e946 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -614,12 +614,12 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - node.banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(subNet, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(subNet); } } else { - node.banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + node.banman->Ban(netAddr, banTime, absolute); if (node.connman) { node.connman->DisconnectNode(netAddr); } @@ -628,7 +628,7 @@ static UniValue setban(const JSONRPCRequest& request) else if(strCommand == "remove") { if (!( isSubnet ? node.banman->Unban(subNet) : node.banman->Unban(netAddr) )) { - throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); + throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously manually banned."); } } return NullUniValue; @@ -637,7 +637,7 @@ static UniValue setban(const JSONRPCRequest& request) static UniValue listbanned(const JSONRPCRequest& request) { RPCHelpMan{"listbanned", - "\nList all banned IPs/Subnets.\n", + "\nList all manually banned IPs/Subnets.\n", {}, RPCResult{RPCResult::Type::ARR, "", "", { @@ -646,7 +646,6 @@ static UniValue listbanned(const JSONRPCRequest& request) {RPCResult::Type::STR, "address", ""}, {RPCResult::Type::NUM_TIME, "banned_until", ""}, {RPCResult::Type::NUM_TIME, "ban_created", ""}, - {RPCResult::Type::STR, "ban_reason", ""}, }}, }}, RPCExamples{ @@ -671,7 +670,6 @@ static UniValue listbanned(const JSONRPCRequest& request) rec.pushKV("address", entry.first.ToString()); rec.pushKV("banned_until", banEntry.nBanUntil); rec.pushKV("ban_created", banEntry.nCreateTime); - rec.pushKV("ban_reason", banEntry.banReasonToString()); bannedAddresses.push_back(rec); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 814f8bddfe..5f8c02df65 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -305,7 +305,7 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) CDataStream ssMB(SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); CMerkleBlock mb(block, setTxids); ssMB << mb; - std::string strHex = HexStr(ssMB.begin(), ssMB.end()); + std::string strHex = HexStr(ssMB); return strHex; } @@ -511,12 +511,12 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) static std::string GetAllOutputTypes() { - std::string ret; - for (int i = TX_NONSTANDARD; i <= TX_WITNESS_UNKNOWN; ++i) { - if (i != TX_NONSTANDARD) ret += ", "; - ret += GetTxnOutputType(static_cast<txnouttype>(i)); + std::vector<std::string> ret; + using U = std::underlying_type<TxoutType>::type; + for (U i = (U)TxoutType::NONSTANDARD; i <= (U)TxoutType::WITNESS_UNKNOWN; ++i) { + ret.emplace_back(GetTxnOutputType(static_cast<TxoutType>(i))); } - return ret; + return Join(ret, ", "); } static UniValue decodescript(const JSONRPCRequest& request) @@ -580,10 +580,10 @@ static UniValue decodescript(const JSONRPCRequest& request) // is a witness program, don't return addresses for a segwit programs. if (type.get_str() == "pubkey" || type.get_str() == "pubkeyhash" || type.get_str() == "multisig" || type.get_str() == "nonstandard") { std::vector<std::vector<unsigned char>> solutions_data; - txnouttype which_type = Solver(script, solutions_data); + TxoutType which_type = Solver(script, solutions_data); // Uncompressed pubkeys cannot be used with segwit checksigs. // If the script contains an uncompressed pubkey, skip encoding of a segwit program. - if ((which_type == TX_PUBKEY) || (which_type == TX_MULTISIG)) { + if ((which_type == TxoutType::PUBKEY) || (which_type == TxoutType::MULTISIG)) { for (const auto& solution : solutions_data) { if ((solution.size() != 1) && !CPubKey(solution).IsCompressed()) { return r; @@ -592,9 +592,9 @@ static UniValue decodescript(const JSONRPCRequest& request) } UniValue sr(UniValue::VOBJ); CScript segwitScr; - if (which_type == TX_PUBKEY) { + if (which_type == TxoutType::PUBKEY) { segwitScr = GetScriptForDestination(WitnessV0KeyHash(Hash160(solutions_data[0].begin(), solutions_data[0].end()))); - } else if (which_type == TX_PUBKEYHASH) { + } else if (which_type == TxoutType::PUBKEYHASH) { segwitScr = GetScriptForDestination(WitnessV0KeyHash(uint160{solutions_data[0]})); } else { // Scripts that are not fit for P2WPKH are encoded as P2WSH. @@ -1104,6 +1104,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) const PSBTInput& input = psbtx.inputs[i]; UniValue in(UniValue::VOBJ); // UTXOs + bool have_a_utxo = false; if (!input.witness_utxo.IsNull()) { const CTxOut& txout = input.witness_utxo; @@ -1121,7 +1122,9 @@ UniValue decodepsbt(const JSONRPCRequest& request) ScriptToUniv(txout.scriptPubKey, o, true); out.pushKV("scriptPubKey", o); in.pushKV("witness_utxo", out); - } else if (input.non_witness_utxo) { + have_a_utxo = true; + } + if (input.non_witness_utxo) { UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); @@ -1132,7 +1135,9 @@ UniValue decodepsbt(const JSONRPCRequest& request) // Hack to just not show fee later have_all_utxos = false; } - } else { + have_a_utxo = true; + } + if (!have_a_utxo) { have_all_utxos = false; } @@ -1186,7 +1191,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) if (!input.final_script_witness.IsNull()) { UniValue txinwitness(UniValue::VARR); for (const auto& item : input.final_script_witness.stack) { - txinwitness.push_back(HexStr(item.begin(), item.end())); + txinwitness.push_back(HexStr(item)); } in.pushKV("final_scriptwitness", txinwitness); } diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index bd82773bf2..1031716b4a 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -138,10 +138,10 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: entry.pushKV("vout", (uint64_t)txin.prevout.n); UniValue witness(UniValue::VARR); for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { - witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + witness.push_back(HexStr(txin.scriptWitness.stack[i])); } entry.pushKV("witness", witness); - entry.pushKV("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end())); + entry.pushKV("scriptSig", HexStr(txin.scriptSig)); entry.pushKV("sequence", (uint64_t)txin.nSequence); entry.pushKV("error", strMessage); vErrorsRet.push_back(entry); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 844f62cbc6..de8791a935 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -20,10 +20,10 @@ #include <mutex> #include <unordered_map> -static RecursiveMutex cs_rpcWarmup; +static Mutex g_rpc_warmup_mutex; static std::atomic<bool> g_rpc_running{false}; -static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; -static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; +static bool fRPCInWarmup GUARDED_BY(g_rpc_warmup_mutex) = true; +static std::string rpcWarmupStatus GUARDED_BY(g_rpc_warmup_mutex) = "RPC server started"; /* Timer-creating functions */ static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ @@ -327,20 +327,20 @@ void RpcInterruptionPoint() void SetRPCWarmupStatus(const std::string& newStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); rpcWarmupStatus = newStatus; } void SetRPCWarmupFinished() { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); assert(fRPCInWarmup); fRPCInWarmup = false; } bool RPCIsInWarmup(std::string *outStatus) { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (outStatus) *outStatus = rpcWarmupStatus; return fRPCInWarmup; @@ -439,7 +439,7 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const { // Return immediately if in warmup { - LOCK(cs_rpcWarmup); + LOCK(g_rpc_warmup_mutex); if (fRPCInWarmup) throw JSONRPCError(RPC_IN_WARMUP, rpcWarmupStatus); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index e7afef4cac..ca73c699c9 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -113,6 +113,23 @@ std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) return ParseHexV(find_value(o, strKey), strKey); } +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type) +{ + if (param.isNull()) { + return default_type; + } else { + std::string hash_type_input = param.get_str(); + + if (hash_type_input == "hash_serialized_2") { + return CoinStatsHashType::HASH_SERIALIZED; + } else if (hash_type_input == "none") { + return CoinStatsHashType::NONE; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%d is not a valid hash_type", hash_type_input)); + } + } +} + std::string HelpExampleCli(const std::string& methodname, const std::string& args) { return "> bitcoin-cli " + methodname + " " + args + "\n"; @@ -224,7 +241,7 @@ public: obj.pushKV("isscript", false); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } @@ -234,7 +251,7 @@ public: obj.pushKV("isscript", true); obj.pushKV("iswitness", true); obj.pushKV("witness_version", 0); - obj.pushKV("witness_program", HexStr(id.begin(), id.end())); + obj.pushKV("witness_program", HexStr(id)); return obj; } diff --git a/src/rpc/util.h b/src/rpc/util.h index 53dce2c397..96dd1ea74a 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_RPC_UTIL_H #define BITCOIN_RPC_UTIL_H +#include <node/coinstats.h> #include <node/transaction.h> #include <outputtype.h> #include <protocol.h> @@ -77,6 +78,8 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); +CoinStatsHashType ParseHashType(const UniValue& param, const CoinStatsHashType default_type); + extern CAmount AmountFromValue(const UniValue& value); extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index c4bd47310b..7c361bf26f 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -30,9 +30,6 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { - if (!shouldStop() && taskQueue.empty()) { - REVERSE_LOCK(lock); - } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); @@ -71,18 +68,6 @@ void CScheduler::serviceQueue() newTaskScheduled.notify_one(); } -void CScheduler::stop(bool drain) -{ - { - LOCK(newTaskMutex); - if (drain) - stopWhenEmpty = true; - else - stopRequested = true; - } - newTaskScheduled.notify_all(); -} - void CScheduler::schedule(CScheduler::Function f, std::chrono::system_clock::time_point t) { { @@ -125,8 +110,8 @@ void CScheduler::scheduleEvery(CScheduler::Function f, std::chrono::milliseconds scheduleFromNow([=] { Repeat(*this, f, delta); }, delta); } -size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point &first, - std::chrono::system_clock::time_point &last) const +size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point& first, + std::chrono::system_clock::time_point& last) const { LOCK(newTaskMutex); size_t result = taskQueue.size(); @@ -137,13 +122,15 @@ size_t CScheduler::getQueueInfo(std::chrono::system_clock::time_point &first, return result; } -bool CScheduler::AreThreadsServicingQueue() const { +bool CScheduler::AreThreadsServicingQueue() const +{ LOCK(newTaskMutex); return nThreadsServicingQueue; } -void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { +void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() +{ { LOCK(m_cs_callbacks_pending); // Try to avoid scheduling too many copies here, but if we @@ -155,8 +142,9 @@ void SingleThreadedSchedulerClient::MaybeScheduleProcessQueue() { m_pscheduler->schedule(std::bind(&SingleThreadedSchedulerClient::ProcessQueue, this), std::chrono::system_clock::now()); } -void SingleThreadedSchedulerClient::ProcessQueue() { - std::function<void ()> callback; +void SingleThreadedSchedulerClient::ProcessQueue() +{ + std::function<void()> callback; { LOCK(m_cs_callbacks_pending); if (m_are_callbacks_running) return; @@ -172,7 +160,8 @@ void SingleThreadedSchedulerClient::ProcessQueue() { struct RAIICallbacksRunning { SingleThreadedSchedulerClient* instance; explicit RAIICallbacksRunning(SingleThreadedSchedulerClient* _instance) : instance(_instance) {} - ~RAIICallbacksRunning() { + ~RAIICallbacksRunning() + { { LOCK(instance->m_cs_callbacks_pending); instance->m_are_callbacks_running = false; @@ -184,7 +173,8 @@ void SingleThreadedSchedulerClient::ProcessQueue() { callback(); } -void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void ()> func) { +void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void()> func) +{ assert(m_pscheduler); { @@ -194,7 +184,8 @@ void SingleThreadedSchedulerClient::AddToProcessQueue(std::function<void ()> fun MaybeScheduleProcessQueue(); } -void SingleThreadedSchedulerClient::EmptyQueue() { +void SingleThreadedSchedulerClient::EmptyQueue() +{ assert(!m_pscheduler->AreThreadsServicingQueue()); bool should_continue = true; while (should_continue) { @@ -204,7 +195,8 @@ void SingleThreadedSchedulerClient::EmptyQueue() { } } -size_t SingleThreadedSchedulerClient::CallbacksPending() { +size_t SingleThreadedSchedulerClient::CallbacksPending() +{ LOCK(m_cs_callbacks_pending); return m_callbacks_pending.size(); } diff --git a/src/scheduler.h b/src/scheduler.h index 1e64195484..d7fe00d1b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -5,11 +5,6 @@ #ifndef BITCOIN_SCHEDULER_H #define BITCOIN_SCHEDULER_H -// -// NOTE: -// boost::thread should be ported to std::thread -// when we support C++11. -// #include <condition_variable> #include <functional> #include <list> @@ -17,24 +12,23 @@ #include <sync.h> -// -// Simple class for background tasks that should be run -// periodically or once "after a while" -// -// Usage: -// -// CScheduler* s = new CScheduler(); -// s->scheduleFromNow(doSomething, std::chrono::milliseconds{11}); // Assuming a: void doSomething() { } -// s->scheduleFromNow([=] { this->func(argument); }, std::chrono::milliseconds{3}); -// boost::thread* t = new boost::thread(std::bind(CScheduler::serviceQueue, s)); -// -// ... then at program shutdown, make sure to call stop() to clean up the thread(s) running serviceQueue: -// s->stop(); -// t->join(); -// delete t; -// delete s; // Must be done after thread is interrupted/joined. -// - +/** + * Simple class for background tasks that should be run + * periodically or once "after a while" + * + * Usage: + * + * CScheduler* s = new CScheduler(); + * s->scheduleFromNow(doSomething, std::chrono::milliseconds{11}); // Assuming a: void doSomething() { } + * s->scheduleFromNow([=] { this->func(argument); }, std::chrono::milliseconds{3}); + * std::thread* t = new std::thread([&] { s->serviceQueue(); }); + * + * ... then at program shutdown, make sure to call stop() to clean up the thread(s) running serviceQueue: + * s->stop(); + * t->join(); + * delete t; + * delete s; // Must be done after thread is interrupted/joined. + */ class CScheduler { public: @@ -43,7 +37,7 @@ public: typedef std::function<void()> Function; - // Call func at/after time t + /** Call func at/after time t */ void schedule(Function f, std::chrono::system_clock::time_point t); /** Call f once after the delta has passed */ @@ -67,23 +61,33 @@ public: */ void MockForward(std::chrono::seconds delta_seconds); - // To keep things as simple as possible, there is no unschedule. - - // Services the queue 'forever'. Should be run in a thread, - // and interrupted using boost::interrupt_thread + /** + * Services the queue 'forever'. Should be run in a thread, + * and interrupted using boost::interrupt_thread + */ void serviceQueue(); - // Tell any threads running serviceQueue to stop as soon as they're - // done servicing whatever task they're currently servicing (drain=false) - // or when there is no work left to be done (drain=true) - void stop(bool drain=false); + /** Tell any threads running serviceQueue to stop as soon as the current task is done */ + void stop() + { + WITH_LOCK(newTaskMutex, stopRequested = true); + newTaskScheduled.notify_all(); + } + /** Tell any threads running serviceQueue to stop when there is no work left to be done */ + void StopWhenDrained() + { + WITH_LOCK(newTaskMutex, stopWhenEmpty = true); + newTaskScheduled.notify_all(); + } - // Returns number of tasks waiting to be serviced, - // and first and last task times - size_t getQueueInfo(std::chrono::system_clock::time_point &first, - std::chrono::system_clock::time_point &last) const; + /** + * Returns number of tasks waiting to be serviced, + * and first and last task times + */ + size_t getQueueInfo(std::chrono::system_clock::time_point& first, + std::chrono::system_clock::time_point& last) const; - // Returns true if there are threads actively running in serviceQueue() + /** Returns true if there are threads actively running in serviceQueue() */ bool AreThreadsServicingQueue() const; private: @@ -106,19 +110,20 @@ private: * B() will be able to observe all of the effects of callback A() which executed * before it. */ -class SingleThreadedSchedulerClient { +class SingleThreadedSchedulerClient +{ private: - CScheduler *m_pscheduler; + CScheduler* m_pscheduler; RecursiveMutex m_cs_callbacks_pending; - std::list<std::function<void ()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); + std::list<std::function<void()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); bool m_are_callbacks_running GUARDED_BY(m_cs_callbacks_pending) = false; void MaybeScheduleProcessQueue(); void ProcessQueue(); public: - explicit SingleThreadedSchedulerClient(CScheduler *pschedulerIn) : m_pscheduler(pschedulerIn) {} + explicit SingleThreadedSchedulerClient(CScheduler* pschedulerIn) : m_pscheduler(pschedulerIn) {} /** * Add a callback to be executed. Callbacks are executed serially @@ -126,10 +131,12 @@ public: * Practically, this means that callbacks can behave as if they are executed * in order by a single thread. */ - void AddToProcessQueue(std::function<void ()> func); + void AddToProcessQueue(std::function<void()> func); - // Processes all remaining queue members on the calling thread, blocking until queue is empty - // Must be called after the CScheduler has no remaining processing threads! + /** + * Processes all remaining queue members on the calling thread, blocking until queue is empty + * Must be called after the CScheduler has no remaining processing threads! + */ void EmptyQueue(); size_t CallbacksPending(); diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 7a5421ab6f..5fa128d62d 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -235,7 +235,7 @@ public: } bool IsRange() const override { return false; } size_t GetSize() const override { return m_pubkey.size(); } - std::string ToString() const override { return HexStr(m_pubkey.begin(), m_pubkey.end()); } + std::string ToString() const override { return HexStr(m_pubkey); } bool ToPrivateString(const SigningProvider& arg, std::string& ret) const override { CKey key; @@ -583,7 +583,7 @@ class RawDescriptor final : public DescriptorImpl { const CScript m_script; protected: - std::string ToStringExtra() const override { return HexStr(m_script.begin(), m_script.end()); } + std::string ToStringExtra() const override { return HexStr(m_script); } std::vector<CScript> MakeScripts(const std::vector<CPubKey>&, const CScript*, FlatSigningProvider&) const override { return Vector(m_script); } public: RawDescriptor(CScript script) : DescriptorImpl({}, {}, "raw"), m_script(std::move(script)) {} @@ -985,15 +985,15 @@ std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptCo std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { std::vector<std::vector<unsigned char>> data; - txnouttype txntype = Solver(script, data); + TxoutType txntype = Solver(script, data); - if (txntype == TX_PUBKEY) { + if (txntype == TxoutType::PUBKEY) { CPubKey pubkey(data[0].begin(), data[0].end()); if (pubkey.IsValid()) { return MakeUnique<PKDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_PUBKEYHASH) { + if (txntype == TxoutType::PUBKEYHASH) { uint160 hash(data[0]); CKeyID keyid(hash); CPubKey pubkey; @@ -1001,7 +1001,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<PKHDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_WITNESS_V0_KEYHASH && ctx != ParseScriptContext::P2WSH) { + if (txntype == TxoutType::WITNESS_V0_KEYHASH && ctx != ParseScriptContext::P2WSH) { uint160 hash(data[0]); CKeyID keyid(hash); CPubKey pubkey; @@ -1009,7 +1009,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<WPKHDescriptor>(InferPubkey(pubkey, ctx, provider)); } } - if (txntype == TX_MULTISIG) { + if (txntype == TxoutType::MULTISIG) { std::vector<std::unique_ptr<PubkeyProvider>> providers; for (size_t i = 1; i + 1 < data.size(); ++i) { CPubKey pubkey(data[i].begin(), data[i].end()); @@ -1017,7 +1017,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } return MakeUnique<MultisigDescriptor>((int)data[0][0], std::move(providers)); } - if (txntype == TX_SCRIPTHASH && ctx == ParseScriptContext::TOP) { + if (txntype == TxoutType::SCRIPTHASH && ctx == ParseScriptContext::TOP) { uint160 hash(data[0]); CScriptID scriptid(hash); CScript subscript; @@ -1026,7 +1026,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo if (sub) return MakeUnique<SHDescriptor>(std::move(sub)); } } - if (txntype == TX_WITNESS_V0_SCRIPTHASH && ctx != ParseScriptContext::P2WSH) { + if (txntype == TxoutType::WITNESS_V0_SCRIPTHASH && ctx != ParseScriptContext::P2WSH) { CScriptID scriptid; CRIPEMD160().Write(data[0].data(), data[0].size()).Finalize(scriptid.begin()); CScript subscript; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 43988c4fd7..f425215549 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -92,11 +92,11 @@ static bool CreateSig(const BaseSignatureCreator& creator, SignatureData& sigdat /** * Sign scriptPubKey using signature made with creator. * Signatures are returned in scriptSigRet (or returns false if scriptPubKey can't be signed), - * unless whichTypeRet is TX_SCRIPTHASH, in which case scriptSigRet is the redemption script. + * unless whichTypeRet is TxoutType::SCRIPTHASH, in which case scriptSigRet is the redemption script. * Returns false if scriptPubKey could not be completely satisfied. */ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator& creator, const CScript& scriptPubKey, - std::vector<valtype>& ret, txnouttype& whichTypeRet, SigVersion sigversion, SignatureData& sigdata) + std::vector<valtype>& ret, TxoutType& whichTypeRet, SigVersion sigversion, SignatureData& sigdata) { CScript scriptRet; uint160 h160; @@ -108,15 +108,15 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator switch (whichTypeRet) { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::NULL_DATA: + case TxoutType::WITNESS_UNKNOWN: return false; - case TX_PUBKEY: + case TxoutType::PUBKEY: if (!CreateSig(creator, sigdata, provider, sig, CPubKey(vSolutions[0]), scriptPubKey, sigversion)) return false; ret.push_back(std::move(sig)); return true; - case TX_PUBKEYHASH: { + case TxoutType::PUBKEYHASH: { CKeyID keyID = CKeyID(uint160(vSolutions[0])); CPubKey pubkey; if (!GetPubKey(provider, sigdata, keyID, pubkey)) { @@ -129,7 +129,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(ToByteVector(pubkey)); return true; } - case TX_SCRIPTHASH: + case TxoutType::SCRIPTHASH: h160 = uint160(vSolutions[0]); if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); @@ -139,7 +139,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator sigdata.missing_redeem_script = h160; return false; - case TX_MULTISIG: { + case TxoutType::MULTISIG: { size_t required = vSolutions.front()[0]; ret.push_back(valtype()); // workaround CHECKMULTISIG bug for (size_t i = 1; i < vSolutions.size() - 1; ++i) { @@ -159,11 +159,11 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator } return ok; } - case TX_WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_KEYHASH: ret.push_back(vSolutions[0]); return true; - case TX_WITNESS_V0_SCRIPTHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(h160.begin()); if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); @@ -198,44 +198,44 @@ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreato if (sigdata.complete) return true; std::vector<valtype> result; - txnouttype whichType; + TxoutType whichType; bool solved = SignStep(provider, creator, fromPubKey, result, whichType, SigVersion::BASE, sigdata); bool P2SH = false; CScript subscript; sigdata.scriptWitness.stack.clear(); - if (solved && whichType == TX_SCRIPTHASH) + if (solved && whichType == TxoutType::SCRIPTHASH) { // Solver returns the subscript that needs to be evaluated; // the final scriptSig is the signatures from that // and then the serialized subscript: subscript = CScript(result[0].begin(), result[0].end()); sigdata.redeem_script = subscript; - solved = solved && SignStep(provider, creator, subscript, result, whichType, SigVersion::BASE, sigdata) && whichType != TX_SCRIPTHASH; + solved = solved && SignStep(provider, creator, subscript, result, whichType, SigVersion::BASE, sigdata) && whichType != TxoutType::SCRIPTHASH; P2SH = true; } - if (solved && whichType == TX_WITNESS_V0_KEYHASH) + if (solved && whichType == TxoutType::WITNESS_V0_KEYHASH) { CScript witnessscript; witnessscript << OP_DUP << OP_HASH160 << ToByteVector(result[0]) << OP_EQUALVERIFY << OP_CHECKSIG; - txnouttype subType; + TxoutType subType; solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata); sigdata.scriptWitness.stack = result; sigdata.witness = true; result.clear(); } - else if (solved && whichType == TX_WITNESS_V0_SCRIPTHASH) + else if (solved && whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { CScript witnessscript(result[0].begin(), result[0].end()); sigdata.witness_script = witnessscript; - txnouttype subType; - solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata) && subType != TX_SCRIPTHASH && subType != TX_WITNESS_V0_SCRIPTHASH && subType != TX_WITNESS_V0_KEYHASH; + TxoutType subType; + solved = solved && SignStep(provider, creator, witnessscript, result, subType, SigVersion::WITNESS_V0, sigdata) && subType != TxoutType::SCRIPTHASH && subType != TxoutType::WITNESS_V0_SCRIPTHASH && subType != TxoutType::WITNESS_V0_KEYHASH; result.push_back(std::vector<unsigned char>(witnessscript.begin(), witnessscript.end())); sigdata.scriptWitness.stack = result; sigdata.witness = true; result.clear(); - } else if (solved && whichType == TX_WITNESS_UNKNOWN) { + } else if (solved && whichType == TxoutType::WITNESS_UNKNOWN) { sigdata.witness = true; } @@ -301,11 +301,11 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI // Get scripts std::vector<std::vector<unsigned char>> solutions; - txnouttype script_type = Solver(txout.scriptPubKey, solutions); + TxoutType script_type = Solver(txout.scriptPubKey, solutions); SigVersion sigversion = SigVersion::BASE; CScript next_script = txout.scriptPubKey; - if (script_type == TX_SCRIPTHASH && !stack.script.empty() && !stack.script.back().empty()) { + if (script_type == TxoutType::SCRIPTHASH && !stack.script.empty() && !stack.script.back().empty()) { // Get the redeemScript CScript redeem_script(stack.script.back().begin(), stack.script.back().end()); data.redeem_script = redeem_script; @@ -315,7 +315,7 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI script_type = Solver(next_script, solutions); stack.script.pop_back(); } - if (script_type == TX_WITNESS_V0_SCRIPTHASH && !stack.witness.empty() && !stack.witness.back().empty()) { + if (script_type == TxoutType::WITNESS_V0_SCRIPTHASH && !stack.witness.empty() && !stack.witness.back().empty()) { // Get the witnessScript CScript witness_script(stack.witness.back().begin(), stack.witness.back().end()); data.witness_script = witness_script; @@ -328,7 +328,7 @@ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nI stack.witness.clear(); sigversion = SigVersion::WITNESS_V0; } - if (script_type == TX_MULTISIG && !stack.script.empty()) { + if (script_type == TxoutType::MULTISIG && !stack.script.empty()) { // Build a map of pubkey -> signature by matching sigs to pubkeys: assert(solutions.size() > 1); unsigned int num_pubkeys = solutions.size()-2; @@ -454,13 +454,13 @@ bool IsSegWitOutput(const SigningProvider& provider, const CScript& script) { std::vector<valtype> solutions; auto whichtype = Solver(script, solutions); - if (whichtype == TX_WITNESS_V0_SCRIPTHASH || whichtype == TX_WITNESS_V0_KEYHASH || whichtype == TX_WITNESS_UNKNOWN) return true; - if (whichtype == TX_SCRIPTHASH) { + if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; + if (whichtype == TxoutType::SCRIPTHASH) { auto h160 = uint160(solutions[0]); CScript subscript; if (provider.GetCScript(CScriptID{h160}, subscript)) { whichtype = Solver(subscript, solutions); - if (whichtype == TX_WITNESS_V0_SCRIPTHASH || whichtype == TX_WITNESS_V0_KEYHASH || whichtype == TX_WITNESS_UNKNOWN) return true; + if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; } } return false; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 2adf6ce56d..1c4990791c 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -43,20 +43,20 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) CSHA256().Write(in.data(), in.size()).Finalize(begin()); } -std::string GetTxnOutputType(txnouttype t) +std::string GetTxnOutputType(TxoutType t) { switch (t) { - case TX_NONSTANDARD: return "nonstandard"; - case TX_PUBKEY: return "pubkey"; - case TX_PUBKEYHASH: return "pubkeyhash"; - case TX_SCRIPTHASH: return "scripthash"; - case TX_MULTISIG: return "multisig"; - case TX_NULL_DATA: return "nulldata"; - case TX_WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; - case TX_WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; - case TX_WITNESS_UNKNOWN: return "witness_unknown"; - } + case TxoutType::NONSTANDARD: return "nonstandard"; + case TxoutType::PUBKEY: return "pubkey"; + case TxoutType::PUBKEYHASH: return "pubkeyhash"; + case TxoutType::SCRIPTHASH: return "scripthash"; + case TxoutType::MULTISIG: return "multisig"; + case TxoutType::NULL_DATA: return "nulldata"; + case TxoutType::WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; + case TxoutType::WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; + case TxoutType::WITNESS_UNKNOWN: return "witness_unknown"; + } // no default case, so the compiler can warn about missing cases assert(false); } @@ -106,7 +106,7 @@ static bool MatchMultisig(const CScript& script, unsigned int& required, std::ve return (it + 1 == script.end()); } -txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) { vSolutionsRet.clear(); @@ -116,7 +116,7 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned { std::vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); vSolutionsRet.push_back(hashBytes); - return TX_SCRIPTHASH; + return TxoutType::SCRIPTHASH; } int witnessversion; @@ -124,18 +124,18 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_KEYHASH_SIZE) { vSolutionsRet.push_back(witnessprogram); - return TX_WITNESS_V0_KEYHASH; + return TxoutType::WITNESS_V0_KEYHASH; } if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_SCRIPTHASH_SIZE) { vSolutionsRet.push_back(witnessprogram); - return TX_WITNESS_V0_SCRIPTHASH; + return TxoutType::WITNESS_V0_SCRIPTHASH; } if (witnessversion != 0) { vSolutionsRet.push_back(std::vector<unsigned char>{(unsigned char)witnessversion}); vSolutionsRet.push_back(std::move(witnessprogram)); - return TX_WITNESS_UNKNOWN; + return TxoutType::WITNESS_UNKNOWN; } - return TX_NONSTANDARD; + return TxoutType::NONSTANDARD; } // Provably prunable, data-carrying output @@ -144,18 +144,18 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned // byte passes the IsPushOnly() test we don't care what exactly is in the // script. if (scriptPubKey.size() >= 1 && scriptPubKey[0] == OP_RETURN && scriptPubKey.IsPushOnly(scriptPubKey.begin()+1)) { - return TX_NULL_DATA; + return TxoutType::NULL_DATA; } std::vector<unsigned char> data; if (MatchPayToPubkey(scriptPubKey, data)) { vSolutionsRet.push_back(std::move(data)); - return TX_PUBKEY; + return TxoutType::PUBKEY; } if (MatchPayToPubkeyHash(scriptPubKey, data)) { vSolutionsRet.push_back(std::move(data)); - return TX_PUBKEYHASH; + return TxoutType::PUBKEYHASH; } unsigned int required; @@ -164,19 +164,19 @@ txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..16 vSolutionsRet.insert(vSolutionsRet.end(), keys.begin(), keys.end()); vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..16 - return TX_MULTISIG; + return TxoutType::MULTISIG; } vSolutionsRet.clear(); - return TX_NONSTANDARD; + return TxoutType::NONSTANDARD; } bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) { std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); + TxoutType whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_PUBKEY) { + if (whichType == TxoutType::PUBKEY) { CPubKey pubKey(vSolutions[0]); if (!pubKey.IsValid()) return false; @@ -184,26 +184,26 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) addressRet = PKHash(pubKey); return true; } - else if (whichType == TX_PUBKEYHASH) + else if (whichType == TxoutType::PUBKEYHASH) { addressRet = PKHash(uint160(vSolutions[0])); return true; } - else if (whichType == TX_SCRIPTHASH) + else if (whichType == TxoutType::SCRIPTHASH) { addressRet = ScriptHash(uint160(vSolutions[0])); return true; - } else if (whichType == TX_WITNESS_V0_KEYHASH) { + } else if (whichType == TxoutType::WITNESS_V0_KEYHASH) { WitnessV0KeyHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TX_WITNESS_V0_SCRIPTHASH) { + } else if (whichType == TxoutType::WITNESS_V0_SCRIPTHASH) { WitnessV0ScriptHash hash; std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); addressRet = hash; return true; - } else if (whichType == TX_WITNESS_UNKNOWN) { + } else if (whichType == TxoutType::WITNESS_UNKNOWN) { WitnessUnknown unk; unk.version = vSolutions[0][0]; std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); @@ -215,19 +215,19 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) return false; } -bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) +bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) { addressRet.clear(); std::vector<valtype> vSolutions; typeRet = Solver(scriptPubKey, vSolutions); - if (typeRet == TX_NONSTANDARD) { + if (typeRet == TxoutType::NONSTANDARD) { return false; - } else if (typeRet == TX_NULL_DATA) { + } else if (typeRet == TxoutType::NULL_DATA) { // This is data, not addresses return false; } - if (typeRet == TX_MULTISIG) + if (typeRet == TxoutType::MULTISIG) { nRequiredRet = vSolutions.front()[0]; for (unsigned int i = 1; i < vSolutions.size()-1; i++) @@ -290,14 +290,11 @@ public: return CScript() << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); } }; - -const CScriptVisitor g_script_visitor; - } // namespace CScript GetScriptForDestination(const CTxDestination& dest) { - return boost::apply_visitor(::g_script_visitor, dest); + return boost::apply_visitor(CScriptVisitor(), dest); } CScript GetScriptForRawPubKey(const CPubKey& pubKey) @@ -319,10 +316,10 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) CScript GetScriptForWitness(const CScript& redeemscript) { std::vector<std::vector<unsigned char> > vSolutions; - txnouttype typ = Solver(redeemscript, vSolutions); - if (typ == TX_PUBKEY) { + TxoutType typ = Solver(redeemscript, vSolutions); + if (typ == TxoutType::PUBKEY) { return GetScriptForDestination(WitnessV0KeyHash(Hash160(vSolutions[0].begin(), vSolutions[0].end()))); - } else if (typ == TX_PUBKEYHASH) { + } else if (typ == TxoutType::PUBKEYHASH) { return GetScriptForDestination(WitnessV0KeyHash(uint160{vSolutions[0]})); } return GetScriptForDestination(WitnessV0ScriptHash(redeemscript)); diff --git a/src/script/standard.h b/src/script/standard.h index 4baed6da6e..fd29353886 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -99,11 +99,11 @@ static const unsigned int MAX_OP_RETURN_RELAY = 83; /** * A data carrying output is an unspendable output containing data. The script - * type is designated as TX_NULL_DATA. + * type is designated as TxoutType::NULL_DATA. */ extern bool fAcceptDatacarrier; -/** Maximum size of TX_NULL_DATA scripts that this node considers standard. */ +/** Maximum size of TxoutType::NULL_DATA scripts that this node considers standard. */ extern unsigned nMaxDatacarrierBytes; /** @@ -116,18 +116,17 @@ extern unsigned nMaxDatacarrierBytes; */ static const unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS = SCRIPT_VERIFY_P2SH; -enum txnouttype -{ - TX_NONSTANDARD, +enum class TxoutType { + NONSTANDARD, // 'standard' transaction types: - TX_PUBKEY, - TX_PUBKEYHASH, - TX_SCRIPTHASH, - TX_MULTISIG, - TX_NULL_DATA, //!< unspendable OP_RETURN script that carries data - TX_WITNESS_V0_SCRIPTHASH, - TX_WITNESS_V0_KEYHASH, - TX_WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above + PUBKEY, + PUBKEYHASH, + SCRIPTHASH, + MULTISIG, + NULL_DATA, //!< unspendable OP_RETURN script that carries data + WITNESS_V0_SCRIPTHASH, + WITNESS_V0_KEYHASH, + WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above }; class CNoDestination { @@ -200,11 +199,11 @@ struct WitnessUnknown /** * A txout script template with a specific destination. It is either: * * CNoDestination: no destination set - * * PKHash: TX_PUBKEYHASH destination (P2PKH) - * * ScriptHash: TX_SCRIPTHASH destination (P2SH) - * * WitnessV0ScriptHash: TX_WITNESS_V0_SCRIPTHASH destination (P2WSH) - * * WitnessV0KeyHash: TX_WITNESS_V0_KEYHASH destination (P2WPKH) - * * WitnessUnknown: TX_WITNESS_UNKNOWN destination (P2W???) + * * PKHash: TxoutType::PUBKEYHASH destination (P2PKH) + * * ScriptHash: TxoutType::SCRIPTHASH destination (P2SH) + * * WitnessV0ScriptHash: TxoutType::WITNESS_V0_SCRIPTHASH destination (P2WSH) + * * WitnessV0KeyHash: TxoutType::WITNESS_V0_KEYHASH destination (P2WPKH) + * * WitnessUnknown: TxoutType::WITNESS_UNKNOWN destination (P2W???) * A CTxDestination is the internal data type encoded in a bitcoin address */ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown> CTxDestination; @@ -212,8 +211,8 @@ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, /** Check whether a CTxDestination is a CNoDestination. */ bool IsValidDestination(const CTxDestination& dest); -/** Get the name of a txnouttype as a C string, or nullptr if unknown. */ -std::string GetTxnOutputType(txnouttype t); +/** Get the name of a TxoutType as a string */ +std::string GetTxnOutputType(TxoutType t); /** * Parse a scriptPubKey and identify script type for standard scripts. If @@ -223,9 +222,9 @@ std::string GetTxnOutputType(txnouttype t); * * @param[in] scriptPubKey Script to parse * @param[out] vSolutionsRet Vector of parsed pubkeys and hashes - * @return The script type. TX_NONSTANDARD represents a failed solve. + * @return The script type. TxoutType::NONSTANDARD represents a failed solve. */ -txnouttype Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); /** * Parse a standard scriptPubKey for the destination address. Assigns result to @@ -246,7 +245,7 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) * encodable as an address) with key identifiers (of keys involved in a * CScript), and its use should be phased out. */ -bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); +bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); /** * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH diff --git a/src/span.h b/src/span.h index 4931507719..841f1eadf7 100644 --- a/src/span.h +++ b/src/span.h @@ -21,6 +21,62 @@ /** A Span is an object that can refer to a contiguous sequence of objects. * * It implements a subset of C++20's std::span. + * + * Things to be aware of when writing code that deals with Spans: + * + * - Similar to references themselves, Spans are subject to reference lifetime + * issues. The user is responsible for making sure the objects pointed to by + * a Span live as long as the Span is used. For example: + * + * std::vector<int> vec{1,2,3,4}; + * Span<int> sp(vec); + * vec.push_back(5); + * printf("%i\n", sp.front()); // UB! + * + * may exhibit undefined behavior, as increasing the size of a vector may + * invalidate references. + * + * - One particular pitfall is that Spans can be constructed from temporaries, + * but this is unsafe when the Span is stored in a variable, outliving the + * temporary. For example, this will compile, but exhibits undefined behavior: + * + * Span<const int> sp(std::vector<int>{1, 2, 3}); + * printf("%i\n", sp.front()); // UB! + * + * The lifetime of the vector ends when the statement it is created in ends. + * Thus the Span is left with a dangling reference, and using it is undefined. + * + * - Due to Span's automatic creation from range-like objects (arrays, and data + * types that expose a data() and size() member function), functions that + * accept a Span as input parameter can be called with any compatible + * range-like object. For example, this works: +* + * void Foo(Span<const int> arg); + * + * Foo(std::vector<int>{1, 2, 3}); // Works + * + * This is very useful in cases where a function truly does not care about the + * container, and only about having exactly a range of elements. However it + * may also be surprising to see automatic conversions in this case. + * + * When a function accepts a Span with a mutable element type, it will not + * accept temporaries; only variables or other references. For example: + * + * void FooMut(Span<int> arg); + * + * FooMut(std::vector<int>{1, 2, 3}); // Does not compile + * std::vector<int> baz{1, 2, 3}; + * FooMut(baz); // Works + * + * This is similar to how functions that take (non-const) lvalue references + * as input cannot accept temporaries. This does not work either: + * + * void FooVec(std::vector<int>& arg); + * FooVec(std::vector<int>{1, 2, 3}); // Does not compile + * + * The idea is that if a function accepts a mutable reference, a meaningful + * result will be present in that variable after the call. Passing a temporary + * is useless in that context. */ template<typename C> class Span diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 7dff2e6e86..00c4bdc14e 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -94,7 +94,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, CBlockHeader header = block->GetBlockHeader(); BlockValidationState state; - if (!EnsureChainman(m_node).ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { + if (!Assert(m_node.chainman)->ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainA_last_header = last_header; for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainB_last_header = last_header; for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; @@ -220,7 +220,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // Reorg back to chain A. for (size_t i = 2; i < 4; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)); } // Check that chain A and B blocks can be retrieved. diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 60196c36a5..173ec5e3d9 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -538,7 +538,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream tmp(SER_DISK, CLIENT_VERSION); uint64_t x = 3000000000ULL; tmp << VARINT(x); - BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); + BOOST_CHECK_EQUAL(HexStr(tmp), "8a95c0bb00"); CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { Coin cc5; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 348b170536..1fe01fae04 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -238,8 +238,8 @@ BOOST_AUTO_TEST_CASE(DoS_banning) LOCK2(cs_main, dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(banman->IsBanned(addr1)); - BOOST_CHECK(!banman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned + BOOST_CHECK(banman->IsDiscouraged(addr1)); + BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); @@ -255,8 +255,8 @@ BOOST_AUTO_TEST_CASE(DoS_banning) LOCK2(cs_main, dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(!banman->IsBanned(addr2)); // 2 not banned yet... - BOOST_CHECK(banman->IsBanned(addr1)); // ... but 1 still should be + BOOST_CHECK(!banman->IsDiscouraged(addr2)); // 2 not banned yet... + BOOST_CHECK(banman->IsDiscouraged(addr1)); // ... but 1 still should be { LOCK(cs_main); Misbehaving(dummyNode2.GetId(), 50); @@ -265,7 +265,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) LOCK2(cs_main, dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } - BOOST_CHECK(banman->IsBanned(addr2)); + BOOST_CHECK(banman->IsDiscouraged(addr2)); bool dummy; peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); @@ -294,7 +294,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) LOCK2(cs_main, dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(!banman->IsBanned(addr1)); + BOOST_CHECK(!banman->IsDiscouraged(addr1)); { LOCK(cs_main); Misbehaving(dummyNode1.GetId(), 10); @@ -303,7 +303,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) LOCK2(cs_main, dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(!banman->IsBanned(addr1)); + BOOST_CHECK(!banman->IsDiscouraged(addr1)); { LOCK(cs_main); Misbehaving(dummyNode1.GetId(), 1); @@ -312,7 +312,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) LOCK2(cs_main, dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); } - BOOST_CHECK(banman->IsBanned(addr1)); + BOOST_CHECK(banman->IsDiscouraged(addr1)); gArgs.ForceSetArg("-banscore", ToString(DEFAULT_BANSCORE_THRESHOLD)); bool dummy; @@ -344,13 +344,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) LOCK2(cs_main, dummyNode.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); } - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60); - BOOST_CHECK(banman->IsBanned(addr)); - - SetMockTime(nStartTime+60*60*24+1); - BOOST_CHECK(!banman->IsBanned(addr)); + BOOST_CHECK(banman->IsDiscouraged(addr)); bool dummy; peerLogic->FinalizeNode(dummyNode.GetId(), dummy); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 5f9a78ceb2..5d7065dafb 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -216,7 +216,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st // For each of the produced scripts, verify solvability, and when possible, try to sign a transaction spending it. for (size_t n = 0; n < spks.size(); ++n) { - BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n].begin(), spks[n].end())); + BOOST_CHECK_EQUAL(ref[n], HexStr(spks[n])); BOOST_CHECK_EQUAL(IsSolvable(Merge(key_provider, script_provider), spks[n]), (flags & UNSOLVABLE) == 0); if (flags & SIGNABLE) { diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index 524cea83fe..ad6461650f 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -18,18 +18,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CBanEntry ban_entry = [&] { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { case 0: return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; break; - case 1: - return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>(), fuzzed_data_provider.PickValueInArray<BanReason>({ - BanReason::BanReasonUnknown, - BanReason::BanReasonNodeMisbehaving, - BanReason::BanReasonManuallyAdded, - })}; - break; - case 2: { + case 1: { const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); if (ban_entry) { return *ban_entry; @@ -39,5 +32,4 @@ void test_one_input(const std::vector<uint8_t>& buffer) } return CBanEntry{}; }(); - assert(!ban_entry.banReasonToString().empty()); } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 52dd62a145..c186bef7ae 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -278,7 +278,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CCoinsStats stats; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, stats); + (void)GetUTXOStats(&coins_view_cache, stats, CoinStatsHashType::HASH_SERIALIZED); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp new file mode 100644 index 0000000000..595cdf9abb --- /dev/null +++ b/src/test/fuzz/crypto.cpp @@ -0,0 +1,124 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/hmac_sha256.h> +#include <crypto/hmac_sha512.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> +#include <hash.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + + CHash160 hash160; + CHash256 hash256; + CHMAC_SHA256 hmac_sha256{data.data(), data.size()}; + CHMAC_SHA512 hmac_sha512{data.data(), data.size()}; + CRIPEMD160 ripemd160; + CSHA1 sha1; + CSHA256 sha256; + CSHA512 sha512; + CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; + + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { + case 0: { + if (fuzzed_data_provider.ConsumeBool()) { + data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + } + + (void)hash160.Write(data.data(), data.size()); + (void)hash256.Write(data.data(), data.size()); + (void)hmac_sha256.Write(data.data(), data.size()); + (void)hmac_sha512.Write(data.data(), data.size()); + (void)ripemd160.Write(data.data(), data.size()); + (void)sha1.Write(data.data(), data.size()); + (void)sha256.Write(data.data(), data.size()); + (void)sha512.Write(data.data(), data.size()); + (void)sip_hasher.Write(data.data(), data.size()); + + (void)Hash(data.begin(), data.end()); + (void)Hash160(data); + (void)Hash160(data.begin(), data.end()); + (void)sha512.Size(); + break; + } + case 1: { + (void)hash160.Reset(); + (void)hash256.Reset(); + (void)ripemd160.Reset(); + (void)sha1.Reset(); + (void)sha256.Reset(); + (void)sha512.Reset(); + break; + } + case 2: { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 8)) { + case 0: { + data.resize(CHash160::OUTPUT_SIZE); + hash160.Finalize(data.data()); + break; + } + case 1: { + data.resize(CHash256::OUTPUT_SIZE); + hash256.Finalize(data.data()); + break; + } + case 2: { + data.resize(CHMAC_SHA256::OUTPUT_SIZE); + hmac_sha256.Finalize(data.data()); + break; + } + case 3: { + data.resize(CHMAC_SHA512::OUTPUT_SIZE); + hmac_sha512.Finalize(data.data()); + break; + } + case 4: { + data.resize(CRIPEMD160::OUTPUT_SIZE); + ripemd160.Finalize(data.data()); + break; + } + case 5: { + data.resize(CSHA1::OUTPUT_SIZE); + sha1.Finalize(data.data()); + break; + } + case 6: { + data.resize(CSHA256::OUTPUT_SIZE); + sha256.Finalize(data.data()); + break; + } + case 7: { + data.resize(CSHA512::OUTPUT_SIZE); + sha512.Finalize(data.data()); + break; + } + case 8: { + data.resize(1); + data[0] = sip_hasher.Finalize() % 256; + break; + } + } + break; + } + } + } +} diff --git a/src/test/fuzz/decode_tx.cpp b/src/test/fuzz/decode_tx.cpp index 09c4ff05df..0d89d4228a 100644 --- a/src/test/fuzz/decode_tx.cpp +++ b/src/test/fuzz/decode_tx.cpp @@ -14,7 +14,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { - const std::string tx_hex = HexStr(std::string{buffer.begin(), buffer.end()}); + const std::string tx_hex = HexStr(buffer); CMutableTransaction mtx; const bool result_none = DecodeHexTx(mtx, tx_hex, false, false); const bool result_try_witness = DecodeHexTx(mtx, tx_hex, false, true); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 82e1d55c0b..1e1807d734 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -12,7 +12,16 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; -#if defined(__AFL_COMPILER) +// Decide if main(...) should be provided: +// * AFL needs main(...) regardless of platform. +// * macOS handles __attribute__((weak)) main(...) poorly when linking +// against libFuzzer. See https://github.com/bitcoin/bitcoin/pull/18008 +// for details. +#if defined(__AFL_COMPILER) || !defined(MAC_OSX) +#define PROVIDE_MAIN_FUNCTION +#endif + +#if defined(PROVIDE_MAIN_FUNCTION) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -44,9 +53,8 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Generally, the fuzzer will provide main(), except for AFL -#if defined(__AFL_COMPILER) -int main(int argc, char** argv) +#if defined(PROVIDE_MAIN_FUNCTION) +__attribute__((weak)) int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 1919a5f881..c746374c61 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -108,7 +108,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(pubkey.IsCompressed()); assert(pubkey.IsValid()); assert(pubkey.IsFullyValid()); - assert(HexToPubKey(HexStr(pubkey.begin(), pubkey.end())) == pubkey); + assert(HexToPubKey(HexStr(pubkey)) == pubkey); assert(GetAllDestinationsForKey(pubkey).size() == 3); } @@ -157,25 +157,25 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(ok_add_key_pubkey); assert(fillable_signing_provider_pub.HaveKey(pubkey.GetID())); - txnouttype which_type_tx_pubkey; + TxoutType which_type_tx_pubkey; const bool is_standard_tx_pubkey = IsStandard(tx_pubkey_script, which_type_tx_pubkey); assert(is_standard_tx_pubkey); - assert(which_type_tx_pubkey == txnouttype::TX_PUBKEY); + assert(which_type_tx_pubkey == TxoutType::PUBKEY); - txnouttype which_type_tx_multisig; + TxoutType which_type_tx_multisig; const bool is_standard_tx_multisig = IsStandard(tx_multisig_script, which_type_tx_multisig); assert(is_standard_tx_multisig); - assert(which_type_tx_multisig == txnouttype::TX_MULTISIG); + assert(which_type_tx_multisig == TxoutType::MULTISIG); std::vector<std::vector<unsigned char>> v_solutions_ret_tx_pubkey; - const txnouttype outtype_tx_pubkey = Solver(tx_pubkey_script, v_solutions_ret_tx_pubkey); - assert(outtype_tx_pubkey == txnouttype::TX_PUBKEY); + const TxoutType outtype_tx_pubkey = Solver(tx_pubkey_script, v_solutions_ret_tx_pubkey); + assert(outtype_tx_pubkey == TxoutType::PUBKEY); assert(v_solutions_ret_tx_pubkey.size() == 1); assert(v_solutions_ret_tx_pubkey[0].size() == 33); std::vector<std::vector<unsigned char>> v_solutions_ret_tx_multisig; - const txnouttype outtype_tx_multisig = Solver(tx_multisig_script, v_solutions_ret_tx_multisig); - assert(outtype_tx_multisig == txnouttype::TX_MULTISIG); + const TxoutType outtype_tx_multisig = Solver(tx_multisig_script, v_solutions_ret_tx_multisig); + assert(outtype_tx_multisig == TxoutType::MULTISIG); assert(v_solutions_ret_tx_multisig.size() == 3); assert(v_solutions_ret_tx_multisig[0].size() == 1); assert(v_solutions_ret_tx_multisig[1].size() == 33); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 64328fb66e..908e2b16f2 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -39,7 +39,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)psbt.IsNull(); - (void)psbt.IsSane(); Optional<CMutableTransaction> tx = psbt.tx; if (tx) { @@ -50,7 +49,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) for (const PSBTInput& input : psbt.inputs) { (void)PSBTInputSigned(input); (void)input.IsNull(); - (void)input.IsSane(); } for (const PSBTOutput& output : psbt.outputs) { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 933cf9049d..cad548178d 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -58,7 +58,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CTxDestination address; (void)ExtractDestination(script, address); - txnouttype type_ret; + TxoutType type_ret; std::vector<CTxDestination> addresses; int required_ret; (void)ExtractDestinations(script, type_ret, addresses, required_ret); @@ -72,7 +72,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)IsSolvable(signing_provider, script); - txnouttype which_type; + TxoutType which_type; (void)IsStandard(script, which_type); (void)RecursiveDynamicUsage(script); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index cf2bd03698..fd35537c77 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -5,6 +5,7 @@ #include <key.h> #include <key_io.h> +#include <streams.h> #include <test/util/setup_common.h> #include <uint256.h> #include <util/strencodings.h> @@ -220,4 +221,47 @@ BOOST_AUTO_TEST_CASE(key_key_negation) BOOST_CHECK(key.GetPubKey().data()[0] == 0x03); } +static CPubKey UnserializePubkey(const std::vector<uint8_t>& data) +{ + CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + stream << data; + CPubKey pubkey; + stream >> pubkey; + return pubkey; +} + +static unsigned int GetLen(unsigned char chHeader) +{ + if (chHeader == 2 || chHeader == 3) + return CPubKey::COMPRESSED_SIZE; + if (chHeader == 4 || chHeader == 6 || chHeader == 7) + return CPubKey::SIZE; + return 0; +} + +static void CmpSerializationPubkey(const CPubKey& pubkey) +{ + CDataStream stream{SER_NETWORK, INIT_PROTO_VERSION}; + stream << pubkey; + CPubKey pubkey2; + stream >> pubkey2; + BOOST_CHECK(pubkey == pubkey2); +} + +BOOST_AUTO_TEST_CASE(pubkey_unserialize) +{ + for (uint8_t i = 2; i <= 7; ++i) { + CPubKey key = UnserializePubkey({0x02}); + BOOST_CHECK(!key.IsValid()); + CmpSerializationPubkey(key); + key = UnserializePubkey(std::vector<uint8_t>(GetLen(i), i)); + CmpSerializationPubkey(key); + if (i == 5) { + BOOST_CHECK(!key.IsValid()); + } else { + BOOST_CHECK(key.IsValid()); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 11ff7b833b..62a0dc4241 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nNonce = blockinfo[i].nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index dd2890c134..e14d2dd72d 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(multisig_IsStandard) for (int i = 0; i < 4; i++) key[i].MakeNewKey(true); - txnouttype whichType; + TxoutType whichType; CScript a_and_b; a_and_b << OP_2 << ToByteVector(key[0].GetPubKey()) << ToByteVector(key[1].GetPubKey()) << OP_2 << OP_CHECKMULTISIG; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index d0ec401f9d..0fbf257f0e 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -160,6 +160,9 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(ResolveSubNet("1.2.2.20/26").Match(ResolveIP("1.2.2.63"))); // All-Matching IPv6 Matches arbitrary IPv4 and IPv6 BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); + // But not `::` or `0.0.0.0` because they are considered invalid addresses + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("::"))); + BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("0.0.0.0"))); BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); // All-Matching IPv4 does not Match IPv6 BOOST_CHECK(!ResolveSubNet("0.0.0.0/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); diff --git a/src/test/policy_fee_tests.cpp b/src/test/policy_fee_tests.cpp new file mode 100644 index 0000000000..6d8872b11e --- /dev/null +++ b/src/test/policy_fee_tests.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <policy/fees.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(policy_fee_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(FeeRounder) +{ + FeeFilterRounder fee_rounder{CFeeRate{1000}}; + + // check that 1000 rounds to 974 or 1071 + std::set<CAmount> results; + while (results.size() < 2) { + results.emplace(fee_rounder.round(1000)); + } + BOOST_CHECK_EQUAL(*results.begin(), 974); + BOOST_CHECK_EQUAL(*++results.begin(), 1071); + + // check that negative amounts rounds to 0 + BOOST_CHECK_EQUAL(fee_rounder.round(-0), 0); + BOOST_CHECK_EQUAL(fee_rounder.round(-1), 0); + + // check that MAX_MONEY rounds to 9170997 + BOOST_CHECK_EQUAL(fee_rounder.round(MAX_MONEY), 9170997); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index fcee6a9b9d..2e5a7549b7 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -89,7 +89,7 @@ BOOST_AUTO_TEST_CASE(manythreads) } // Drain the task queue then exit threads - microTasks.stop(true); + microTasks.StopWhenDrained(); microThreads.join_all(); // ... wait until all the threads are done int counterSum = 0; @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) } // finish up - scheduler.stop(true); + scheduler.StopWhenDrained(); threads.join_all(); BOOST_CHECK_EQUAL(counter1, 100); @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(mockforward) scheduler.MockForward(std::chrono::minutes{5}); // ensure scheduler has chance to process all tasks queued for before 1 ms from now. - scheduler.scheduleFromNow([&scheduler] { scheduler.stop(false); }, std::chrono::milliseconds{1}); + scheduler.scheduleFromNow([&scheduler] { scheduler.stop(); }, std::chrono::milliseconds{1}); scheduler_thread.join(); // check that the queue only has one job remaining diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index b185d3b4ac..77d748241b 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -31,35 +31,35 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) CScript s; std::vector<std::vector<unsigned char> > solutions; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_PUBKEY); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::PUBKEY); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0])); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_PUBKEYHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0].GetID())); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_SCRIPTHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(CScriptID(redeemScript))); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_1 << ToByteVector(pubkeys[0]) << ToByteVector(pubkeys[1]) << OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_MULTISIG); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::MULTISIG); BOOST_CHECK_EQUAL(solutions.size(), 4U); BOOST_CHECK(solutions[0] == std::vector<unsigned char>({1})); BOOST_CHECK(solutions[1] == ToByteVector(pubkeys[0])); @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) ToByteVector(pubkeys[1]) << ToByteVector(pubkeys[2]) << OP_3 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_MULTISIG); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::MULTISIG); BOOST_CHECK_EQUAL(solutions.size(), 5U); BOOST_CHECK(solutions[0] == std::vector<unsigned char>({2})); BOOST_CHECK(solutions[1] == ToByteVector(pubkeys[0])); @@ -80,37 +80,37 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_CHECK(solutions[3] == ToByteVector(pubkeys[2])); BOOST_CHECK(solutions[4] == std::vector<unsigned char>({3})); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({0}) << std::vector<unsigned char>({75}) << std::vector<unsigned char>({255}); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NULL_DATA); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NULL_DATA); BOOST_CHECK_EQUAL(solutions.size(), 0U); - // TX_WITNESS_V0_KEYHASH + // TxoutType::WITNESS_V0_KEYHASH s.clear(); s << OP_0 << ToByteVector(pubkeys[0].GetID()); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_WITNESS_V0_KEYHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V0_KEYHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(pubkeys[0].GetID())); - // TX_WITNESS_V0_SCRIPTHASH + // TxoutType::WITNESS_V0_SCRIPTHASH uint256 scriptHash; CSHA256().Write(&redeemScript[0], redeemScript.size()) .Finalize(scriptHash.begin()); s.clear(); s << OP_0 << ToByteVector(scriptHash); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_WITNESS_V0_SCRIPTHASH); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::WITNESS_V0_SCRIPTHASH); BOOST_CHECK_EQUAL(solutions.size(), 1U); BOOST_CHECK(solutions[0] == ToByteVector(scriptHash)); - // TX_NONSTANDARD + // TxoutType::NONSTANDARD s.clear(); s << OP_9 << OP_ADD << OP_11 << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); } BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) @@ -123,50 +123,50 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) CScript s; std::vector<std::vector<unsigned char> > solutions; - // TX_PUBKEY with incorrectly sized pubkey + // TxoutType::PUBKEY with incorrectly sized pubkey s.clear(); s << std::vector<unsigned char>(30, 0x01) << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_PUBKEYHASH with incorrectly sized key hash + // TxoutType::PUBKEYHASH with incorrectly sized key hash s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_SCRIPTHASH with incorrectly sized script hash + // TxoutType::SCRIPTHASH with incorrectly sized script hash s.clear(); s << OP_HASH160 << std::vector<unsigned char>(21, 0x01) << OP_EQUAL; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG 0/2 + // TxoutType::MULTISIG 0/2 s.clear(); s << OP_0 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG 2/1 + // TxoutType::MULTISIG 2/1 s.clear(); s << OP_2 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG n = 2 with 1 pubkey + // TxoutType::MULTISIG n = 2 with 1 pubkey s.clear(); s << OP_1 << ToByteVector(pubkey) << OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_MULTISIG n = 1 with 0 pubkeys + // TxoutType::MULTISIG n = 1 with 0 pubkeys s.clear(); s << OP_1 << OP_1 << OP_CHECKMULTISIG; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_NULL_DATA with other opcodes + // TxoutType::NULL_DATA with other opcodes s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}) << OP_ADD; - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); - // TX_WITNESS with incorrect program size + // TxoutType::WITNESS_UNKNOWN with incorrect program size s.clear(); s << OP_0 << std::vector<unsigned char>(19, 0x01); - BOOST_CHECK_EQUAL(Solver(s, solutions), TX_NONSTANDARD); + BOOST_CHECK_EQUAL(Solver(s, solutions), TxoutType::NONSTANDARD); } BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) @@ -179,21 +179,21 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CScript s; CTxDestination address; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkey) << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<PKHash>(&address) && *boost::get<PKHash>(&address) == PKHash(pubkey)); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<PKHash>(&address) && *boost::get<PKHash>(&address) == PKHash(pubkey)); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; @@ -201,17 +201,17 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(boost::get<ScriptHash>(&address) && *boost::get<ScriptHash>(&address) == ScriptHash(redeemScript)); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_1 << ToByteVector(pubkey) << OP_1 << OP_CHECKMULTISIG; BOOST_CHECK(!ExtractDestination(s, address)); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}); BOOST_CHECK(!ExtractDestination(s, address)); - // TX_WITNESS_V0_KEYHASH + // TxoutType::WITNESS_V0_KEYHASH s.clear(); s << OP_0 << ToByteVector(pubkey.GetID()); BOOST_CHECK(ExtractDestination(s, address)); @@ -219,7 +219,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CHash160().Write(pubkey.begin(), pubkey.size()).Finalize(keyhash.begin()); BOOST_CHECK(boost::get<WitnessV0KeyHash>(&address) && *boost::get<WitnessV0KeyHash>(&address) == keyhash); - // TX_WITNESS_V0_SCRIPTHASH + // TxoutType::WITNESS_V0_SCRIPTHASH s.clear(); WitnessV0ScriptHash scripthash; CSHA256().Write(redeemScript.data(), redeemScript.size()).Finalize(scripthash.begin()); @@ -227,7 +227,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(ExtractDestination(s, address)); BOOST_CHECK(boost::get<WitnessV0ScriptHash>(&address) && *boost::get<WitnessV0ScriptHash>(&address) == scripthash); - // TX_WITNESS with unknown version + // TxoutType::WITNESS_UNKNOWN with unknown version s.clear(); s << OP_1 << ToByteVector(pubkey); BOOST_CHECK(ExtractDestination(s, address)); @@ -248,49 +248,49 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) } CScript s; - txnouttype whichType; + TxoutType whichType; std::vector<CTxDestination> addresses; int nRequired; - // TX_PUBKEY + // TxoutType::PUBKEY s.clear(); s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_PUBKEY); + BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - // TX_PUBKEYHASH + // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_PUBKEYHASH); + BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - // TX_SCRIPTHASH + // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_SCRIPTHASH); + BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); BOOST_CHECK(boost::get<ScriptHash>(&addresses[0]) && *boost::get<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); - // TX_MULTISIG + // TxoutType::MULTISIG s.clear(); s << OP_2 << ToByteVector(pubkeys[0]) << ToByteVector(pubkeys[1]) << OP_2 << OP_CHECKMULTISIG; BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TX_MULTISIG); + BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); BOOST_CHECK_EQUAL(addresses.size(), 2U); BOOST_CHECK_EQUAL(nRequired, 2); BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && @@ -298,7 +298,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK(boost::get<PKHash>(&addresses[1]) && *boost::get<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); - // TX_NULL_DATA + // TxoutType::NULL_DATA s.clear(); s << OP_RETURN << std::vector<unsigned char>({75}); BOOST_CHECK(!ExtractDestinations(s, whichType, addresses, nRequired)); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 5ca136ea6e..c0bb92258b 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(sighash_test) ss << txTo; std::cout << "\t[\"" ; - std::cout << HexStr(ss.begin(), ss.end()) << "\", \""; + std::cout << HexStr(ss) << "\", \""; std::cout << HexStr(scriptCode) << "\", "; std::cout << nIn << ", "; std::cout << nHashType << ", \""; diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index ddbc68f8e2..4bf6e734ce 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -716,12 +716,12 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "scriptpubkey"); - // MAX_OP_RETURN_RELAY-byte TX_NULL_DATA (standard) + // MAX_OP_RETURN_RELAY-byte TxoutType::NULL_DATA (standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY, t.vout[0].scriptPubKey.size()); BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); - // MAX_OP_RETURN_RELAY+1-byte TX_NULL_DATA (non-standard) + // MAX_OP_RETURN_RELAY+1-byte TxoutType::NULL_DATA (non-standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); reason.clear(); @@ -745,12 +745,12 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "scriptpubkey"); - // TX_NULL_DATA w/o PUSHDATA + // TxoutType::NULL_DATA w/o PUSHDATA t.vout.resize(1); t.vout[0].scriptPubKey = CScript() << OP_RETURN; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); - // Only one TX_NULL_DATA permitted in all cases + // Only one TxoutType::NULL_DATA permitted in all cases t.vout.resize(2); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index dac7f1a07b..74536ae74c 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -11,6 +11,7 @@ #include <node/context.h> #include <pow.h> #include <script/standard.h> +#include <util/check.h> #include <validation.h> CTxIn generatetoaddress(const NodeContext& node, const std::string& address) @@ -31,7 +32,7 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) assert(block->nNonce); } - bool processed{EnsureChainman(node).ProcessNewBlock(Params(), block, true, nullptr)}; + bool processed{Assert(node.chainman)->ProcessNewBlock(Params(), block, true, nullptr)}; assert(processed); return CTxIn{block->vtx[0]->GetHash(), 0}; @@ -39,9 +40,8 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { - assert(node.mempool); auto block = std::make_shared<CBlock>( - BlockAssembler{*node.mempool, Params()} + BlockAssembler{*Assert(node.mempool), Params()} .CreateNewBlock(coinbase_scriptPubKey) ->block); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 3b7a7c8d12..24c0d6382b 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -19,6 +19,7 @@ #include <rpc/blockchain.h> #include <rpc/register.h> #include <rpc/server.h> +#include <scheduler.h> #include <script/sigcache.h> #include <streams.h> #include <txdb.h> @@ -74,11 +75,13 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve "dummy", "-printtoconsole=0", "-logtimemicros", + "-logthreadnames", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", }, extra_args); + util::ThreadRename("test"); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); @@ -129,7 +132,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. - threadGroup.create_thread([&]{ m_node.scheduler->serviceQueue(); }); + threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -228,7 +231,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); CBlock result = block; return result; diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index d5cda8a95b..78b279e42a 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -11,8 +11,8 @@ #include <node/context.h> #include <pubkey.h> #include <random.h> -#include <scheduler.h> #include <txmempool.h> +#include <util/check.h> #include <util/string.h> #include <type_traits> diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index 1beddd334b..6f2faeec6c 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -22,8 +22,8 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int n CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit); // Helper: create two dummy transactions, each with two outputs. -// The first has nValues[0] and nValues[1] outputs paid to a TX_PUBKEY, -// the second nValues[2] and nValues[3] outputs paid to a TX_PUBKEYHASH. +// The first has nValues[0] and nValues[1] outputs paid to a TxoutType::PUBKEY, +// the second nValues[2] and nValues[3] outputs paid to a TxoutType::PUBKEYHASH. std::vector<CMutableTransaction> SetupDummyInputs(FillableSigningProvider& keystoreRet, CCoinsViewCache& coinsRet, const std::array<CAmount,4>& nValues); #endif // BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 4dcc080b2d..f3f9fb2bff 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -53,8 +53,6 @@ std::set<std::string> RenameEnMasse(int num_threads) */ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) { - BOOST_CHECK_EQUAL(util::ThreadGetInternalName(), ""); - #if !defined(HAVE_THREAD_LOCAL) // This test doesn't apply to platforms where we don't have thread_local. return; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 45e0c5484e..8e85b7df3e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -163,10 +163,10 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlockHeaders(headers, state, Params())); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -188,13 +188,13 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) FastRandomContext insecure; for (int i = 0; i < 1000; i++) { auto block = blocks[insecure.randrange(blocks.size() - 1)]; - EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); + bool processed = Assert(m_node.chainman)->ProcessNewBlock(Params(), block, true, &ignored); assert(processed); } } @@ -233,7 +233,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { - return EnsureChainman(m_node).ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + return Assert(m_node.chainman)->ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); }; // Process all mined blocks diff --git a/src/timedata.cpp b/src/timedata.cpp index 16dac24a48..6b3a79017b 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -9,8 +9,8 @@ #include <timedata.h> #include <netaddress.h> +#include <node/ui_interface.h> #include <sync.h> -#include <ui_interface.h> #include <util/system.h> #include <util/translation.h> #include <warnings.h> diff --git a/src/txdb.cpp b/src/txdb.cpp index 6f652c1375..047560f45d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,10 +5,10 @@ #include <txdb.h> +#include <node/ui_interface.h> #include <pow.h> #include <random.h> #include <shutdown.h> -#include <ui_interface.h> #include <uint256.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/util/check.h b/src/util/check.h index 5c0f32cf51..3d534fd33e 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -25,7 +25,7 @@ class NonFatalCheckError : public std::runtime_error * - where the condition is assumed to be true, not for error handling or validating user input * - where a failure to fulfill the condition is recoverable and does not abort the program * - * For example in RPC code, where it is undersirable to crash the whole program, this can be generally used to replace + * For example in RPC code, where it is undesirable to crash the whole program, this can be generally used to replace * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC * caller, which can then report the issue to the developers. */ @@ -42,4 +42,18 @@ class NonFatalCheckError : public std::runtime_error } \ } while (false) +#if defined(NDEBUG) +#error "Cannot compile without assertions!" +#endif + +/** Helper for Assert(). TODO remove in C++14 and replace `decltype(get_pure_r_value(val))` with `T` (templated lambda) */ +template <typename T> +T get_pure_r_value(T&& val) +{ + return std::forward<T>(val); +} + +/** Identity function. Abort if the value compares equal to zero */ +#define Assert(val) [&]() -> decltype(get_pure_r_value(val))& { auto& check = (val); assert(#val && check); return check; }() + #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/fees.cpp b/src/util/fees.cpp index b335bfa666..6208a20a97 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -6,11 +6,16 @@ #include <util/fees.h> #include <policy/fees.h> +#include <util/strencodings.h> +#include <util/string.h> #include <map> #include <string> +#include <vector> +#include <utility> -std::string StringForFeeReason(FeeReason reason) { +std::string StringForFeeReason(FeeReason reason) +{ static const std::map<FeeReason, std::string> fee_reason_strings = { {FeeReason::NONE, "None"}, {FeeReason::HALF_ESTIMATE, "Half Target 60% Threshold"}, @@ -29,16 +34,31 @@ std::string StringForFeeReason(FeeReason reason) { return reason_string->second; } -bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) { - static const std::map<std::string, FeeEstimateMode> fee_modes = { - {"UNSET", FeeEstimateMode::UNSET}, - {"ECONOMICAL", FeeEstimateMode::ECONOMICAL}, - {"CONSERVATIVE", FeeEstimateMode::CONSERVATIVE}, +const std::vector<std::pair<std::string, FeeEstimateMode>>& FeeModeMap() +{ + static const std::vector<std::pair<std::string, FeeEstimateMode>> FEE_MODES = { + {"unset", FeeEstimateMode::UNSET}, + {"economical", FeeEstimateMode::ECONOMICAL}, + {"conservative", FeeEstimateMode::CONSERVATIVE}, + {(CURRENCY_UNIT + "/kB"), FeeEstimateMode::BTC_KB}, + {(CURRENCY_ATOM + "/B"), FeeEstimateMode::SAT_B}, }; - auto mode = fee_modes.find(mode_string); + return FEE_MODES; +} - if (mode == fee_modes.end()) return false; +std::string FeeModes(const std::string& delimiter) +{ + return Join(FeeModeMap(), delimiter, [&](const std::pair<std::string, FeeEstimateMode>& i) { return i.first; }); +} - fee_estimate_mode = mode->second; - return true; +bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode) +{ + auto searchkey = ToUpper(mode_string); + for (const auto& pair : FeeModeMap()) { + if (ToUpper(pair.first) == searchkey) { + fee_estimate_mode = pair.second; + return true; + } + } + return false; } diff --git a/src/util/fees.h b/src/util/fees.h index a930c8935a..d52046a44c 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -12,5 +12,6 @@ enum class FeeReason; bool FeeModeFromString(const std::string& mode_string, FeeEstimateMode& fee_estimate_mode); std::string StringForFeeReason(FeeReason reason); +std::string FeeModes(const std::string& delimiter); #endif // BITCOIN_UTIL_FEES_H diff --git a/src/util/ui_change_type.h b/src/util/ui_change_type.h new file mode 100644 index 0000000000..1db761a18d --- /dev/null +++ b/src/util/ui_change_type.h @@ -0,0 +1,15 @@ +// Copyright (c) 2012-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_UI_CHANGE_TYPE_H +#define BITCOIN_UTIL_UI_CHANGE_TYPE_H + +/** General change type (added, updated, removed). */ +enum ChangeType { + CT_NEW, + CT_UPDATED, + CT_DELETED +}; + +#endif // BITCOIN_UTIL_UI_CHANGE_TYPE_H diff --git a/src/validation.cpp b/src/validation.cpp index 8bb03fdb97..edc623b205 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <node/ui_interface.h> #include <optional.h> #include <policy/fees.h> #include <policy/policy.h> @@ -36,9 +37,9 @@ #include <tinyformat.h> #include <txdb.h> #include <txmempool.h> -#include <ui_interface.h> #include <uint256.h> #include <undo.h> +#include <util/check.h> // For NDEBUG compile time check #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> @@ -51,10 +52,6 @@ #include <boost/algorithm/string/replace.hpp> -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - #define MICRO 0.000001 #define MILLI 0.001 @@ -1321,12 +1318,6 @@ bool CChainState::IsInitialBlockDownload() const static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; -BlockMap& BlockIndex() -{ - LOCK(::cs_main); - return g_chainman.m_blockman.m_block_index; -} - static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); @@ -1430,12 +1421,12 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c pindexBestHeader = ::ChainActive().Tip(); } - LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: invalid block=%s height=%d log2_work=%f date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(pindexNew->GetBlockTime())); CBlockIndex *tip = ::ChainActive().Tip(); assert (tip); - LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, + LogPrintf("%s: current best=%s height=%d log2_work=%f date=%s\n", __func__, tip->GetBlockHash().ToString(), ::ChainActive().Height(), log(tip->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(tip->GetBlockTime())); CheckForkWarningConditions(); @@ -2485,7 +2476,7 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar if (nUpgraded > 0) AppendWarning(warning_messages, strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, FormatISO8601DateTime(pindexNew->GetBlockTime()), @@ -5251,10 +5242,10 @@ CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blo return *to_modify; } -CChain& ChainstateManager::ActiveChain() const +CChainState& ChainstateManager::ActiveChainstate() const { assert(m_active_chainstate); - return m_active_chainstate->m_chain; + return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const diff --git a/src/validation.h b/src/validation.h index e403bcb51a..a148dacb7c 100644 --- a/src/validation.h +++ b/src/validation.h @@ -799,7 +799,8 @@ public: std::vector<CChainState*> GetAll(); //! The most-work chain. - CChain& ActiveChain() const; + CChainState& ActiveChainstate() const; + CChain& ActiveChain() const { return ActiveChainstate().m_chain; } int ActiveHeight() const { return ActiveChain().Height(); } CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } @@ -879,15 +880,12 @@ public: /** DEPRECATED! Please use node.chainman instead. May only be used in validation.cpp internally */ extern ChainstateManager g_chainman GUARDED_BY(::cs_main); -/** @returns the most-work valid chainstate. */ +/** Please prefer the identical ChainstateManager::ActiveChainstate */ CChainState& ChainstateActive(); -/** @returns the most-work chain. */ +/** Please prefer the identical ChainstateManager::ActiveChain */ CChain& ChainActive(); -/** @returns the global block index map. */ -BlockMap& BlockIndex(); - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 125bf004e4..5f823d5906 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -292,11 +292,10 @@ BerkeleyBatch::SafeDbt::operator Dbt*() return &m_dbt; } -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) +bool BerkeleyDatabase::Verify(bilingual_str& errorStr) { - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); + fs::path file_path = walletDir / strFile; LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); LogPrintf("Using wallet %s\n", file_path.string()); @@ -306,19 +305,10 @@ bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& return false; } - return true; -} - -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr) -{ - std::string walletFile; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); - fs::path walletDir = env->Directory(); - - if (fs::exists(walletDir / walletFile)) + if (fs::exists(file_path)) { - if (!env->Verify(walletFile)) { - errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), walletFile); + if (!env->Verify(strFile)) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path); return false; } } @@ -335,7 +325,7 @@ void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) } -BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr) +BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr), m_cursor(nullptr) { fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; @@ -442,6 +432,7 @@ void BerkeleyBatch::Close() activeTxn->abort(); activeTxn = nullptr; pdb = nullptr; + CloseCursor(); if (fFlushOnClose) Flush(); @@ -494,13 +485,11 @@ void BerkeleyEnvironment::ReloadDbEnv() Open(true); } -bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) +bool BerkeleyDatabase::Rewrite(const char* pszSkip) { - if (database.IsDummy()) { + if (IsDummy()) { return true; } - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; while (true) { { LOCK(cs_db); @@ -514,7 +503,7 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - BerkeleyBatch db(database, "r"); + BerkeleyBatch db(*this, "r"); std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer @@ -528,17 +517,15 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) fSuccess = false; } - Dbc* pcursor = db.GetCursor(); - if (pcursor) + if (db.StartCursor()) { while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret1 == DB_NOTFOUND) { - pcursor->close(); + bool complete; + bool ret1 = db.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - } else if (ret1 != 0) { - pcursor->close(); + } else if (!ret1) { fSuccess = false; break; } @@ -556,6 +543,8 @@ bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip) if (ret2 > 0) fSuccess = false; } + db.CloseCursor(); + } if (fSuccess) { db.Close(); env->CloseDb(strFile); @@ -624,51 +613,35 @@ void BerkeleyEnvironment::Flush(bool fShutdown) } } -bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) +bool BerkeleyDatabase::PeriodicFlush() { - if (database.IsDummy()) { - return true; - } - bool ret = false; - BerkeleyEnvironment *env = database.env.get(); - const std::string& strFile = database.strFile; + // There's nothing to do for dummy databases. Return true. + if (IsDummy()) return true; + + // Don't flush if we can't acquire the lock. TRY_LOCK(cs_db, lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); - while (mit != env->mapFileUseCount.end()) - { - nRefCount += (*mit).second; - mit++; - } + if (!lockDb) return false; - if (nRefCount == 0) - { - std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); - if (mi != env->mapFileUseCount.end()) - { - LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); - int64_t nStart = GetTimeMillis(); + // Don't flush if any databases are in use + for (auto it = env->mapFileUseCount.begin() ; it != env->mapFileUseCount.end(); it++) { + if ((*it).second > 0) return false; + } - // Flush wallet file so it's self contained - env->CloseDb(strFile); - env->CheckpointLSN(strFile); + // Don't flush if there haven't been any batch writes for this database. + auto it = env->mapFileUseCount.find(strFile); + if (it == env->mapFileUseCount.end()) return false; - env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - ret = true; - } - } - } + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); - return ret; -} + // Flush wallet file so it's self contained + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(it); -bool BerkeleyDatabase::Rewrite(const char* pszSkip) -{ - return BerkeleyBatch::Rewrite(*this, pszSkip); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + + return true; } bool BerkeleyDatabase::Backup(const std::string& strDest) const @@ -738,27 +711,30 @@ void BerkeleyDatabase::ReloadDbEnv() } } -Dbc* BerkeleyBatch::GetCursor() +bool BerkeleyBatch::StartCursor() { + assert(!m_cursor); if (!pdb) - return nullptr; - Dbc* pcursor = nullptr; - int ret = pdb->cursor(nullptr, &pcursor, 0); - if (ret != 0) - return nullptr; - return pcursor; + return false; + int ret = pdb->cursor(nullptr, &m_cursor, 0); + return ret == 0; } -int BerkeleyBatch::ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue) +bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete) { + complete = false; + if (m_cursor == nullptr) return false; // Read at cursor SafeDbt datKey; SafeDbt datValue; - int ret = pcursor->get(datKey, datValue, DB_NEXT); + int ret = m_cursor->get(datKey, datValue, DB_NEXT); + if (ret == DB_NOTFOUND) { + complete = true; + } if (ret != 0) - return ret; + return false; else if (datKey.get_data() == nullptr || datValue.get_data() == nullptr) - return 99999; + return false; // Convert to streams ssKey.SetType(SER_DISK); @@ -767,7 +743,14 @@ int BerkeleyBatch::ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& s ssValue.SetType(SER_DISK); ssValue.clear(); ssValue.write((char*)datValue.get_data(), datValue.get_size()); - return 0; + return true; +} + +void BerkeleyBatch::CloseCursor() +{ + if (!m_cursor) return; + m_cursor->close(); + m_cursor = nullptr; } bool BerkeleyBatch::TxnBegin() @@ -804,15 +787,13 @@ std::string BerkeleyDatabaseVersion() return DbEnv::version(nullptr, nullptr, nullptr); } -bool BerkeleyBatch::ReadKey(CDataStream& key, CDataStream& value) +bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) { if (!pdb) return false; - // Key SafeDbt datKey(key.data(), key.size()); - // Read SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { @@ -822,48 +803,41 @@ bool BerkeleyBatch::ReadKey(CDataStream& key, CDataStream& value) return false; } -bool BerkeleyBatch::WriteKey(CDataStream& key, CDataStream& value, bool overwrite) +bool BerkeleyBatch::WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite) { if (!pdb) return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); - // Key SafeDbt datKey(key.data(), key.size()); - // Value SafeDbt datValue(value.data(), value.size()); - // Write int ret = pdb->put(activeTxn, datKey, datValue, (overwrite ? 0 : DB_NOOVERWRITE)); return (ret == 0); } -bool BerkeleyBatch::EraseKey(CDataStream& key) +bool BerkeleyBatch::EraseKey(CDataStream&& key) { if (!pdb) return false; if (fReadOnly) assert(!"Erase called on database in read-only mode"); - // Key SafeDbt datKey(key.data(), key.size()); - // Erase int ret = pdb->del(activeTxn, datKey, 0); return (ret == 0 || ret == DB_NOTFOUND); } -bool BerkeleyBatch::HasKey(CDataStream& key) +bool BerkeleyBatch::HasKey(CDataStream&& key) { if (!pdb) return false; - // Key SafeDbt datKey(key.data(), key.size()); - // Exists int ret = pdb->exists(activeTxn, datKey, 0); return ret == 0; } diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index c121bb4228..599319482b 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -131,6 +131,9 @@ public: /** Make sure all changes are flushed to disk. */ void Flush(bool shutdown); + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + bool PeriodicFlush(); void IncrementUpdateCounter(); @@ -141,6 +144,9 @@ public: unsigned int nLastFlushed; int64_t nLastWalletUpdate; + /** Verifies the environment and database file */ + bool Verify(bilingual_str& error); + /** * Pointer to shared database environment. * @@ -189,15 +195,16 @@ class BerkeleyBatch }; private: - bool ReadKey(CDataStream& key, CDataStream& value); - bool WriteKey(CDataStream& key, CDataStream& value, bool overwrite=true); - bool EraseKey(CDataStream& key); - bool HasKey(CDataStream& key); + bool ReadKey(CDataStream&& key, CDataStream& value); + bool WriteKey(CDataStream&& key, CDataStream&& value, bool overwrite = true); + bool EraseKey(CDataStream&& key); + bool HasKey(CDataStream&& key); protected: Db* pdb; std::string strFile; DbTxn* activeTxn; + Dbc* m_cursor; bool fReadOnly; bool fFlushOnClose; BerkeleyEnvironment *env; @@ -212,85 +219,63 @@ public: void Flush(); void Close(); - /* flush the wallet passively (TRY_LOCK) - ideal to be called periodically */ - static bool PeriodicFlush(BerkeleyDatabase& database); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr); - template <typename K, typename T> bool Read(const K& key, T& value) { - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; CDataStream ssValue(SER_DISK, CLIENT_VERSION); - bool success = false; - bool ret = ReadKey(ssKey, ssValue); - if (ret) { - // Unserialize value - try { - ssValue >> value; - success = true; - } catch (const std::exception&) { - // In this case success remains 'false' - } + if (!ReadKey(std::move(ssKey), ssValue)) return false; + try { + ssValue >> value; + return true; + } catch (const std::exception&) { + return false; } - return ret && success; } template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - // Value CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(10000); ssValue << value; - // Write - return WriteKey(ssKey, ssValue, fOverwrite); + return WriteKey(std::move(ssKey), std::move(ssValue), fOverwrite); } template <typename K> bool Erase(const K& key) { - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - // Erase - return EraseKey(ssKey); + return EraseKey(std::move(ssKey)); } template <typename K> bool Exists(const K& key) { - // Key CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(1000); ssKey << key; - // Exists - return HasKey(ssKey); + return HasKey(std::move(ssKey)); } - Dbc* GetCursor(); - int ReadAtCursor(Dbc* pcursor, CDataStream& ssKey, CDataStream& ssValue); + bool StartCursor(); + bool ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& complete); + void CloseCursor(); bool TxnBegin(); bool TxnCommit(); bool TxnAbort(); - - bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); }; std::string BerkeleyDatabaseVersion(); diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 3885eb6185..f173b5e62b 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -7,8 +7,8 @@ #include <interfaces/chain.h> #include <net.h> #include <node/context.h> +#include <node/ui_interface.h> #include <outputtype.h> -#include <ui_interface.h> #include <util/moneystr.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 541675bcef..c9ea6c2ad9 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -817,7 +817,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) create_time = FormatISO8601DateTime(it->second.nCreateTime); } if(spk_man.GetCScript(scriptid, script)) { - file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); + file << strprintf("%s %s script=1", HexStr(script), create_time); file << strprintf(" # addr=%s\n", address); } } @@ -856,20 +856,20 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d { // Use Solver to obtain script type and parsed pubkeys or hashes: std::vector<std::vector<unsigned char>> solverdata; - txnouttype script_type = Solver(script, solverdata); + TxoutType script_type = Solver(script, solverdata); switch (script_type) { - case TX_PUBKEY: { + case TxoutType::PUBKEY: { CPubKey pubkey(solverdata[0].begin(), solverdata[0].end()); import_data.used_keys.emplace(pubkey.GetID(), false); return ""; } - case TX_PUBKEYHASH: { + case TxoutType::PUBKEYHASH: { CKeyID id = CKeyID(uint160(solverdata[0])); import_data.used_keys[id] = true; return ""; } - case TX_SCRIPTHASH: { + case TxoutType::SCRIPTHASH: { if (script_ctx == ScriptContext::P2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside another P2SH"); if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside a P2WSH"); CHECK_NONFATAL(script_ctx == ScriptContext::TOP); @@ -880,14 +880,14 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d import_data.import_scripts.emplace(*subscript); return RecurseImportData(*subscript, import_data, ScriptContext::P2SH); } - case TX_MULTISIG: { + case TxoutType::MULTISIG: { for (size_t i = 1; i + 1< solverdata.size(); ++i) { CPubKey pubkey(solverdata[i].begin(), solverdata[i].end()); import_data.used_keys.emplace(pubkey.GetID(), false); } return ""; } - case TX_WITNESS_V0_SCRIPTHASH: { + case TxoutType::WITNESS_V0_SCRIPTHASH: { if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2WSH inside another P2WSH"); uint256 fullid(solverdata[0]); CScriptID id; @@ -901,7 +901,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d import_data.import_scripts.emplace(*subscript); return RecurseImportData(*subscript, import_data, ScriptContext::WITNESS_V0); } - case TX_WITNESS_V0_KEYHASH: { + case TxoutType::WITNESS_V0_KEYHASH: { if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2WPKH inside P2WSH"); CKeyID id = CKeyID(uint160(solverdata[0])); import_data.used_keys[id] = true; @@ -910,10 +910,10 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } return ""; } - case TX_NULL_DATA: + case TxoutType::NULL_DATA: return "unspendable script"; - case TX_NONSTANDARD: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::WITNESS_UNKNOWN: default: return "unrecognized script"; } @@ -1193,7 +1193,7 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con // Check whether we have any work to do for (const CScript& script : script_pub_keys) { if (pwallet->IsMine(script) & ISMINE_SPENDABLE) { - throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script) + "\")"); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 72998c30fd..55114a17d7 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -45,6 +45,8 @@ using interfaces::FoundBlock; static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; +static const uint32_t WALLET_BTC_KB_TO_SAT_B = COIN / 1000; // 1 sat / B = 0.00001 BTC / kB + static inline bool GetAvoidReuseFlag(const CWallet* const pwallet, const UniValue& param) { bool can_avoid_reuse = pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); bool avoid_reuse = param.isNull() ? can_avoid_reuse : param.get_bool(); @@ -191,6 +193,42 @@ static std::string LabelFromValue(const UniValue& value) return label; } +/** + * Update coin control with fee estimation based on the given parameters + * + * @param[in] pwallet Wallet pointer + * @param[in,out] cc Coin control which is to be updated + * @param[in] estimate_mode String value (e.g. "ECONOMICAL") + * @param[in] estimate_param Parameter (blocks to confirm, explicit fee rate, etc) + * @throws a JSONRPCError if estimate_mode is unknown, or if estimate_param is missing when required + */ +static void SetFeeEstimateMode(const CWallet* pwallet, CCoinControl& cc, const UniValue& estimate_mode, const UniValue& estimate_param) +{ + if (!estimate_mode.isNull()) { + if (!FeeModeFromString(estimate_mode.get_str(), cc.m_fee_mode)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); + } + } + + if (cc.m_fee_mode == FeeEstimateMode::BTC_KB || cc.m_fee_mode == FeeEstimateMode::SAT_B) { + if (estimate_param.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Selected estimate_mode requires a fee rate"); + } + + CAmount fee_rate = AmountFromValue(estimate_param); + if (cc.m_fee_mode == FeeEstimateMode::SAT_B) { + fee_rate /= WALLET_BTC_KB_TO_SAT_B; + } + + cc.m_feerate = CFeeRate(fee_rate); + + // default RBF to true for explicit fee rate modes + if (cc.m_signal_bip125_rbf == nullopt) cc.m_signal_bip125_rbf = true; + } else if (!estimate_param.isNull()) { + cc.m_confirm_target = ParseConfirmTarget(estimate_param, pwallet->chain().estimateMaxBlocks()); + } +} + static UniValue getnewaddress(const JSONRPCRequest& request) { RPCHelpMan{"getnewaddress", @@ -268,7 +306,7 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); } - OutputType output_type = pwallet->m_default_change_type != OutputType::CHANGE_AUTO ? pwallet->m_default_change_type : pwallet->m_default_address_type; + OutputType output_type = pwallet->m_default_change_type.get_value_or(pwallet->m_default_address_type); if (!request.params[0].isNull()) { if (!ParseOutputType(request.params[0].get_str(), output_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); @@ -369,11 +407,9 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) {"subtractfeefromamount", RPCArg::Type::BOOL, /* default */ "false", "The fee will be deducted from the amount being sent.\n" " The recipient will receive less bitcoins than you enter in the amount field."}, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "true", "(only available if avoid_reuse wallet flag is set) Avoid spending from dirty addresses; addresses are considered\n" " dirty if they have previously been used in a transaction."}, }, @@ -384,6 +420,8 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1") + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"donation\" \"seans outpost\"") + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" true") + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" false true 0.00002 " + (CURRENCY_UNIT + "/kB")) + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"\" \"\" false true 2 " + (CURRENCY_ATOM + "/B")) + HelpExampleRpc("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\", 0.1, \"donation\", \"seans outpost\"") }, }.Check(request); @@ -425,20 +463,12 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); - } - - if (!request.params[7].isNull()) { - if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } - coin_control.m_avoid_address_reuse = GetAvoidReuseFlag(pwallet, request.params[8]); // We also enable partial spend avoidance if reuse avoidance is set. coin_control.m_avoid_partial_spends |= coin_control.m_avoid_address_reuse; + SetFeeEstimateMode(pwallet, coin_control, request.params[7], request.params[6]); + EnsureWalletIsUnlocked(pwallet); CTransactionRef tx = SendMoney(pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); @@ -780,11 +810,9 @@ static UniValue sendmany(const JSONRPCRequest& request) }, }, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, RPCResult{ RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of\n" @@ -830,15 +858,7 @@ static UniValue sendmany(const JSONRPCRequest& request) coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); - } - - if (!request.params[7].isNull()) { - if (!FeeModeFromString(request.params[7].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } + SetFeeEstimateMode(pwallet, coin_control, request.params[7], request.params[6]); std::set<CTxDestination> destinations; std::vector<CRecipient> vecSend; @@ -964,7 +984,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) UniValue result(UniValue::VOBJ); result.pushKV("address", EncodeDestination(dest)); - result.pushKV("redeemScript", HexStr(inner.begin(), inner.end())); + result.pushKV("redeemScript", HexStr(inner)); result.pushKV("descriptor", descriptor->ToString()); return result; } @@ -2870,7 +2890,7 @@ static UniValue listunspent(const JSONRPCRequest& request) const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); CScript redeemScript; if (provider->GetCScript(hash, redeemScript)) { - entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); + entry.pushKV("redeemScript", HexStr(redeemScript)); // Now check if the redeemScript is actually a P2WSH script CTxDestination witness_destination; if (redeemScript.IsPayToWitnessScriptHash()) { @@ -2882,7 +2902,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + entry.pushKV("witnessScript", HexStr(witnessScript)); } } } @@ -2892,13 +2912,13 @@ static UniValue listunspent(const JSONRPCRequest& request) CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; if (provider->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + entry.pushKV("witnessScript", HexStr(witnessScript)); } } } } - entry.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); entry.pushKV("confirmations", out.nDepth); entry.pushKV("spendable", out.fSpendable); @@ -2973,10 +2993,11 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("changeAddress")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both changeAddress and address_type options"); } - coinControl.m_change_type = pwallet->m_default_change_type; - if (!ParseOutputType(options["change_type"].get_str(), *coinControl.m_change_type)) { + OutputType out_type; + if (!ParseOutputType(options["change_type"].get_str(), out_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown change type '%s'", options["change_type"].get_str())); } + coinControl.m_change_type.emplace(out_type); } coinControl.fAllowWatchOnly = ParseIncludeWatchonly(options["includeWatching"], *pwallet); @@ -2986,6 +3007,12 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("feeRate")) { + if (options.exists("conf_target")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); + } + if (options.exists("estimate_mode")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and feeRate"); + } coinControl.m_feerate = CFeeRate(AmountFromValue(options["feeRate"])); coinControl.fOverrideFeeRate = true; } @@ -2996,20 +3023,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("replaceable")) { coinControl.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - if (options.exists("conf_target")) { - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); - } - coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"], pwallet->chain().estimateMaxBlocks()); - } - if (options.exists("estimate_mode")) { - if (options.exists("feeRate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and feeRate"); - } - if (!FeeModeFromString(options["estimate_mode"].get_str(), coinControl.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } + SetFeeEstimateMode(pwallet, coinControl, options["estimate_mode"], options["conf_target"]); } } else { // if options is null and not a bool @@ -3077,11 +3091,9 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) }, {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, {"iswitness", RPCArg::Type::BOOL, /* default */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction.\n" @@ -3249,8 +3261,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"confTarget", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'confTarget'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, + {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'conf_target'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" " Specify a fee rate instead of relying on the built-in fee estimator.\n" "Must be at least 0.0001 " + CURRENCY_UNIT + " per kB higher than the current transaction fee rate.\n"}, {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" @@ -3260,10 +3272,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" " are replaceable)."}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, }, @@ -3302,15 +3312,24 @@ static UniValue bumpfee(const JSONRPCRequest& request) RPCTypeCheckObj(options, { {"confTarget", UniValueType(UniValue::VNUM)}, + {"conf_target", UniValueType(UniValue::VNUM)}, {"fee_rate", UniValueType(UniValue::VNUM)}, {"replaceable", UniValueType(UniValue::VBOOL)}, {"estimate_mode", UniValueType(UniValue::VSTR)}, }, true, true); - if (options.exists("confTarget") && options.exists("fee_rate")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate."); - } else if (options.exists("confTarget")) { // TODO: alias this to conf_target - coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"], pwallet->chain().estimateMaxBlocks()); + + if (options.exists("confTarget") && options.exists("conf_target")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and conf_target options should not both be set. Use conf_target (confTarget is deprecated)."); + } + + auto conf_target = options.exists("confTarget") ? options["confTarget"] : options["conf_target"]; + + if (!conf_target.isNull()) { + if (options.exists("fee_rate")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate."); + } + coin_control.m_confirm_target = ParseConfirmTarget(conf_target, pwallet->chain().estimateMaxBlocks()); } else if (options.exists("fee_rate")) { CFeeRate fee_rate(AmountFromValue(options["fee_rate"])); if (fee_rate <= CFeeRate(0)) { @@ -3322,11 +3341,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("replaceable")) { coin_control.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - if (options.exists("estimate_mode")) { - if (!FeeModeFromString(options["estimate_mode"].get_str(), coin_control.m_fee_mode)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid estimate_mode parameter"); - } - } + SetFeeEstimateMode(pwallet, coin_control, options["estimate_mode"], conf_target); } // Make sure the results are valid at least up to the most recent block @@ -3489,9 +3504,9 @@ public: { // Always present: script type and redeemscript std::vector<std::vector<unsigned char>> solutions_data; - txnouttype which_type = Solver(subscript, solutions_data); + TxoutType which_type = Solver(subscript, solutions_data); obj.pushKV("script", GetTxnOutputType(which_type)); - obj.pushKV("hex", HexStr(subscript.begin(), subscript.end())); + obj.pushKV("hex", HexStr(subscript)); CTxDestination embedded; if (ExtractDestination(subscript, embedded)) { @@ -3502,18 +3517,18 @@ public: UniValue wallet_detail = boost::apply_visitor(*this, embedded); subobj.pushKVs(wallet_detail); subobj.pushKV("address", EncodeDestination(embedded)); - subobj.pushKV("scriptPubKey", HexStr(subscript.begin(), subscript.end())); + subobj.pushKV("scriptPubKey", HexStr(subscript)); // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. if (subobj.exists("pubkey")) obj.pushKV("pubkey", subobj["pubkey"]); obj.pushKV("embedded", std::move(subobj)); - } else if (which_type == TX_MULTISIG) { + } else if (which_type == TxoutType::MULTISIG) { // Also report some information on multisig scripts (which do not have a corresponding address). // TODO: abstract out the common functionality between this logic and ExtractDestinations. obj.pushKV("sigsrequired", solutions_data[0][0]); UniValue pubkeys(UniValue::VARR); for (size_t i = 1; i < solutions_data.size() - 1; ++i) { CPubKey key(solutions_data[i].begin(), solutions_data[i].end()); - pubkeys.push_back(HexStr(key.begin(), key.end())); + pubkeys.push_back(HexStr(key)); } obj.pushKV("pubkeys", std::move(pubkeys)); } @@ -3669,7 +3684,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("address", currentAddress); CScript scriptPubKey = GetScriptForDestination(dest); - ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + ret.pushKV("scriptPubKey", HexStr(scriptPubKey)); std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); @@ -4016,10 +4031,8 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, {"conf_target", RPCArg::Type::NUM, /* default */ "fall back to wallet's confirmation target (txconfirmtarget)", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, "options"}, {"bip32derivs", RPCArg::Type::BOOL, /* default */ "true", "Include BIP 32 derivation paths for public keys if we know them"}, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index d42950ee42..e6e62332c0 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -20,6 +20,11 @@ bool RecoverDatabaseFile(const fs::path& file_path) std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + if (!env->Open(true /* retry */)) { + tfm::format(std::cerr, "Error initializing wallet database environment %s!", env->Directory()); + return false; + } + // Recovery procedure: // move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index d57f99a205..38d94335a3 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -88,16 +88,16 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s IsMineResult ret = IsMineResult::NO; std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); + TxoutType whichType = Solver(scriptPubKey, vSolutions); CKeyID keyID; switch (whichType) { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: + case TxoutType::NONSTANDARD: + case TxoutType::NULL_DATA: + case TxoutType::WITNESS_UNKNOWN: break; - case TX_PUBKEY: + case TxoutType::PUBKEY: keyID = CPubKey(vSolutions[0]).GetID(); if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { return IsMineResult::INVALID; @@ -106,7 +106,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineResult::SPENDABLE); } break; - case TX_WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_KEYHASH: { if (sigversion == IsMineSigVersion::WITNESS_V0) { // P2WPKH inside P2WSH is invalid. @@ -121,7 +121,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); break; } - case TX_PUBKEYHASH: + case TxoutType::PUBKEYHASH: keyID = CKeyID(uint160(vSolutions[0])); if (!PermitsUncompressed(sigversion)) { CPubKey pubkey; @@ -133,7 +133,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s ret = std::max(ret, IsMineResult::SPENDABLE); } break; - case TX_SCRIPTHASH: + case TxoutType::SCRIPTHASH: { if (sigversion != IsMineSigVersion::TOP) { // P2SH inside P2WSH or P2SH is invalid. @@ -146,7 +146,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s } break; } - case TX_WITNESS_V0_SCRIPTHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: { if (sigversion == IsMineSigVersion::WITNESS_V0) { // P2WSH inside P2WSH is invalid. @@ -165,7 +165,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s break; } - case TX_MULTISIG: + case TxoutType::MULTISIG: { // Never treat bare multisig outputs as ours (they can still be made watchonly-though) if (sigversion == IsMineSigVersion::TOP) { @@ -597,11 +597,6 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; @@ -836,7 +831,7 @@ bool LegacyScriptPubKeyMan::HaveWatchOnly() const static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) { std::vector<std::vector<unsigned char>> solutions; - return Solver(dest, solutions) == TX_PUBKEY && + return Solver(dest, solutions) == TxoutType::PUBKEY && (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); } @@ -1900,8 +1895,8 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ desc_prefix = "wpkh(" + xpub + "/84'"; break; } - default: assert(false); - } + } // no default case, so the compiler can warn about missing cases + assert(!desc_prefix.empty()); // Mainnet derives at 0', testnet and regtest derive at 1' if (Params().IsTestChain()) { @@ -2086,11 +2081,6 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // Get the Sighash type if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { return TransactionError::SIGHASH_MISMATCH; diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index b4c65a8665..ce7e661b67 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -63,8 +63,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Get the final tx CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); - BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + std::string final_hex = HexStr(ssTx); + BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001008a020000000158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e8876500000001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); // Mutate the transaction so that one of the inputs is invalid psbtx.tx->vin[0].prevout.n = 2; diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 97f8c94fa6..10ddfa22ef 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -24,10 +24,10 @@ static void TestPassphraseSingle(const std::vector<unsigned char>& vchSalt, cons if(!correctKey.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchKey.data(), correctKey.data(), crypt.vchKey.size()) == 0, \ - HexStr(crypt.vchKey.begin(), crypt.vchKey.end()) + std::string(" != ") + HexStr(correctKey.begin(), correctKey.end())); + HexStr(crypt.vchKey) + std::string(" != ") + HexStr(correctKey)); if(!correctIV.empty()) BOOST_CHECK_MESSAGE(memcmp(crypt.vchIV.data(), correctIV.data(), crypt.vchIV.size()) == 0, - HexStr(crypt.vchIV.begin(), crypt.vchIV.end()) + std::string(" != ") + HexStr(correctIV.begin(), correctIV.end())); + HexStr(crypt.vchIV) + std::string(" != ") + HexStr(correctIV)); } static void TestPassphrase(const std::vector<unsigned char>& vchSalt, const SecureString& passphrase, uint32_t rounds, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 497ccd14bb..5c565a3d38 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -118,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -144,7 +144,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Prune the remaining block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(newTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(newTip->GetBlockPos().nFile); } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); @@ -181,7 +181,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // Prune the older block file. { LOCK(cs_main); - EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + Assert(m_node.chainman)->PruneOneBlockFile(oldTip->GetBlockPos().nFile); } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); @@ -333,7 +333,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } -static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) +static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) { CMutableTransaction tx; CWalletTx::Confirmation confirm; @@ -341,7 +341,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - auto inserted = ::BlockIndex().emplace(GetRandHash(), new CBlockIndex); + LOCK(cs_main); + auto inserted = chainman.BlockIndex().emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; block = inserted.first->second; @@ -363,24 +364,24 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 BOOST_AUTO_TEST_CASE(ComputeTimeSmart) { // New transaction should use clock time if lower than block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 100, 120), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 100, 120), 100); // Test that updating existing transaction does not change smart time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 1, 200, 220), 100); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 1, 200, 220), 100); // New transaction should use clock time if there's no block time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 2, 300, 0), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 2, 300, 0), 300); // New transaction should use block time if lower than clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 3, 420, 400), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 3, 420, 400), 400); // New transaction should use latest entry time if higher than // min(block time, clock time). - BOOST_CHECK_EQUAL(AddTx(m_wallet, 4, 500, 390), 400); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 4, 500, 390), 400); // If there are future entries, new transaction should use time of the // newest entry that is no more than 300 seconds ahead of the clock time. - BOOST_CHECK_EQUAL(AddTx(m_wallet, 5, 50, 600), 300); + BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 5, 50, 600), 300); // Reset mock time for other tests. SetMockTime(0); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 57eec9baf9..29ff7bbef1 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -99,9 +99,11 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); } +static Mutex g_loading_wallet_mutex; static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; -static std::set<std::string> g_unloading_wallet_set; +static std::set<std::string> g_loading_wallet_set GUARDED_BY(g_loading_wallet_mutex); +static std::set<std::string> g_unloading_wallet_set GUARDED_BY(g_wallet_release_mutex); // Custom deleter for shared_ptr<CWallet>. static void ReleaseWallet(CWallet* wallet) @@ -145,7 +147,8 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +namespace { +std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { if (!CWallet::Verify(chain, location, error, warnings)) { @@ -166,6 +169,19 @@ std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocati return nullptr; } } +} // namespace + +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +{ + auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(location.GetName())); + if (!result.second) { + error = Untranslated("Wallet already being loading."); + return nullptr; + } + auto wallet = LoadWalletInternal(chain, location, error, warnings); + WITH_LOCK(g_loading_wallet_mutex, g_loading_wallet_set.erase(result.first)); + return wallet; +} std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { @@ -2491,13 +2507,8 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp continue; } - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - return TransactionError::INVALID_PSBT; - } - // If we have no utxo, grab it from the wallet. - if (!input.non_witness_utxo && input.witness_utxo.IsNull()) { + if (!input.non_witness_utxo) { const uint256& txhash = txin.prevout.hash; const auto it = mapWallet.find(txhash); if (it != mapWallet.end()) { @@ -2653,11 +2664,11 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin return locktime; } -OutputType CWallet::TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend) +OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) { // If -changetype is specified, always use that change type. - if (change_type != OutputType::CHANGE_AUTO) { - return change_type; + if (change_type) { + return *change_type; } // if m_default_address_type is legacy, use legacy address as change (even @@ -3721,15 +3732,11 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(wallet_path); try { - if (!WalletBatch::VerifyEnvironment(wallet_path, error_string)) { - return false; - } + return database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); return false; } - - return WalletBatch::VerifyDatabaseFile(wallet_path, error_string); } std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) @@ -3826,14 +3833,20 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); - return nullptr; + if (!gArgs.GetArg("-addresstype", "").empty()) { + if (!ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { + error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); + return nullptr; + } } - if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); - return nullptr; + if (!gArgs.GetArg("-changetype", "").empty()) { + OutputType out_type; + if (!ParseOutputType(gArgs.GetArg("-changetype", ""), out_type)) { + error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); + return nullptr; + } + walletInstance->m_default_change_type = out_type; } if (gArgs.IsArgSet("-mintxfee")) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 9931671fb4..32d8481cd8 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -13,11 +13,11 @@ #include <policy/feerate.h> #include <psbt.h> #include <tinyformat.h> -#include <ui_interface.h> #include <util/message.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/ui_change_type.h> #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> @@ -105,9 +105,6 @@ class ReserveDestination; //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; -//! Default for -changetype -constexpr OutputType DEFAULT_CHANGE_TYPE{OutputType::CHANGE_AUTO}; - static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET @@ -934,7 +931,7 @@ public: Balance GetBalance(int min_depth = 0, bool avoid_reuse = true) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; - OutputType TransactionChangeType(OutputType change_type, const std::vector<CRecipient>& vecSend); + OutputType TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend); /** * Insert additional inputs into the transaction by @@ -1012,7 +1009,13 @@ public: CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; - OutputType m_default_change_type{DEFAULT_CHANGE_TYPE}; + /** + * Default output type for change outputs. When unset, automatically choose type + * based on address type setting and the types other of non-change outputs + * (see -changetype option documentation and implementation in + * CWallet::TransactionChangeType for details). + */ + Optional<OutputType> m_default_change_type{}; /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 603887ee58..7da477d5b7 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -700,8 +700,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch.StartCursor()) { pwallet->WalletLogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -712,11 +711,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) + } + else if (!ret) { + m_batch.CloseCursor(); pwallet->WalletLogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -743,10 +745,10 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (!strErr.empty()) pwallet->WalletLogPrintf("%s\n", strErr); } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch.CloseCursor(); // Set the active ScriptPubKeyMans for (auto spk_man_pair : wss.m_active_external_spks) { @@ -850,8 +852,7 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal } // Get cursor - Dbc* pcursor = m_batch.GetCursor(); - if (!pcursor) + if (!m_batch.StartCursor()) { LogPrintf("Error getting wallet database cursor\n"); return DBErrors::CORRUPT; @@ -862,11 +863,12 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = m_batch.ReadAtCursor(pcursor, ssKey, ssValue); - if (ret == DB_NOTFOUND) + bool complete; + bool ret = m_batch.ReadAtCursor(ssKey, ssValue, complete); + if (complete) { break; - else if (ret != 0) - { + } else if (!ret) { + m_batch.CloseCursor(); LogPrintf("Error reading next record from wallet database\n"); return DBErrors::CORRUPT; } @@ -881,10 +883,10 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal ssValue >> vWtx.back(); } } - pcursor->close(); } catch (...) { result = DBErrors::CORRUPT; } + m_batch.CloseCursor(); return result; } @@ -965,7 +967,7 @@ void MaybeCompactWalletDB() } if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { - if (BerkeleyBatch::PeriodicFlush(dbh)) { + if (dbh.PeriodicFlush()) { dbh.nLastFlushed = nUpdateCounter; } } @@ -974,16 +976,6 @@ void MaybeCompactWalletDB() fOneThread = false; } -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); -} - -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr) -{ - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, errorStr); -} - bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { return WriteIC(std::make_pair(DBKeys::DESTDATA, std::make_pair(address, key)), value); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 77ed6beb5d..8a45d81456 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -112,7 +112,7 @@ static bool SalvageWallet(const fs::path& path) // Initialize the environment before recovery bilingual_str error_string; try { - WalletBatch::VerifyEnvironment(path, error_string); + database->Verify(error_string); } catch (const fs::filesystem_error& e) { error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); } @@ -140,11 +140,6 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } - bilingual_str error; - if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); - return false; - } if (command == "info") { std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); diff --git a/test/README.md b/test/README.md index b036a66f67..2341eef00d 100644 --- a/test/README.md +++ b/test/README.md @@ -260,11 +260,11 @@ Use the `-v` option for verbose output. | Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation |-----------|:----------:|:-------------------------------------------:|-------------- -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.700](https://github.com/bitcoin/bitcoin/pull/18210) | `pip3 install mypy==0.700` -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.8.3](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install flake8==3.8.3` +| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.781](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install mypy==0.781` +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.7.1](https://github.com/bitcoin/bitcoin/pull/19348) | [details...](https://github.com/koalaman/shellcheck#installing) | [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.17.1](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install codespell==1.17.1` Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 85e727aff1..5cddd6527e 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -311,15 +311,13 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.start_node(4) # Open most recent wallet in v0.16 (no loadwallet RPC) - self.stop_node(5) - self.start_node(5, extra_args=["-wallet=w2"]) + self.restart_node(5, extra_args=["-wallet=w2"]) wallet = node_v16.get_wallet_rpc("w2") info = wallet.getwalletinfo() assert info['keypoolsize'] == 1 # Create upgrade wallet in v0.16 - self.stop_node(-1) - self.start_node(-1, extra_args=["-wallet=u1_v16"]) + self.restart_node(-1, extra_args=["-wallet=u1_v16"]) wallet = node_v16.get_wallet_rpc("u1_v16") v16_addr = wallet.getnewaddress('', "bech32") v16_info = wallet.validateaddress(v16_addr) diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 82f1331685..0a457ca17f 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -71,8 +71,7 @@ class LoadblockTest(BitcoinTestFramework): check=True) self.log.info("Restart second, unsynced node with bootstrap file") - self.stop_node(1) - self.start_node(1, ["-loadblock=" + bootstrap_file]) + self.restart_node(1, extra_args=["-loadblock=" + bootstrap_file]) assert_equal(self.nodes[1].getblockcount(), 100) # start_node is blocking on all block files being imported assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100) diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index e4bf2d849d..afcbcf099a 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -67,8 +67,7 @@ class LoggingTest(BitcoinTestFramework): assert not os.path.isfile(default_log_path) # just sanity check no crash here - self.stop_node(0) - self.start_node(0, ["-debuglogfile=%s" % os.devnull]) + self.restart_node(0, ["-debuglogfile=%s" % os.devnull]) if __name__ == '__main__': diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index c9362cf5aa..e46e5aacc8 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -263,8 +263,7 @@ class PruneTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "not in prune mode", node.pruneblockchain, 500) # now re-start in manual pruning mode - self.stop_node(node_number) - self.start_node(node_number, extra_args=["-prune=1"]) + self.restart_node(node_number, extra_args=["-prune=1"]) node = self.nodes[node_number] assert_equal(node.getblockcount(), 995) @@ -326,16 +325,14 @@ class PruneTest(BitcoinTestFramework): assert not has_block(3), "blk00003.dat is still there, should be pruned by now" # stop node, start back up with auto-prune at 550 MiB, make sure still runs - self.stop_node(node_number) - self.start_node(node_number, extra_args=["-prune=550"]) + self.restart_node(node_number, extra_args=["-prune=550"]) self.log.info("Success") def wallet_test(self): # check that the pruning node's wallet is still in good shape self.log.info("Stop and start pruning node to trigger wallet rescan") - self.stop_node(2) - self.start_node(2, extra_args=["-prune=550"]) + self.restart_node(2, extra_args=["-prune=550"]) self.log.info("Success") # check that wallet loads successfully when restarting a pruned node after IBD. @@ -344,8 +341,7 @@ class PruneTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 5) nds = [self.nodes[0], self.nodes[5]] self.sync_blocks(nds, wait=5, timeout=300) - self.stop_node(5) # stop and start to trigger rescan - self.start_node(5, extra_args=["-prune=550"]) + self.restart_node(5, extra_args=["-prune=550"]) # restart to trigger rescan self.log.info("Success") def run_test(self): diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 2298485640..5195d20dcb 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -559,8 +559,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) # Assert it is properly saved - self.stop_node(1) - self.start_node(1) + self.restart_node(1) assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 6046237101..5c7e27a3a8 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -49,9 +49,9 @@ class AddrTest(BitcoinTestFramework): addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = msg_addr() - self.log.info('Send too large addr message') + self.log.info('Send too-large addr message') msg.addrs = ADDRS * 101 - with self.nodes[0].assert_debug_log(['message addr size() = 1010']): + with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) self.log.info('Check that addr message content is relayed and added to addrman') diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 9047fc6828..09b9ebeb2d 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -69,8 +69,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setmocktime(old_time + 3) assert_equal(len(self.nodes[1].listbanned()), 3) - self.stop_node(1) - self.start_node(1) + self.restart_node(1) listAfterShutdown = self.nodes[1].listbanned() assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index d99bc621de..d9a9ae5188 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -7,6 +7,9 @@ from test_framework.messages import ( CBlockHeader, CInv, + MAX_HEADERS_RESULTS, + MAX_INV_SIZE, + MAX_PROTOCOL_MESSAGE_LENGTH, msg_getdata, msg_headers, msg_inv, @@ -24,8 +27,7 @@ from test_framework.util import ( wait_until, ) -MSG_LIMIT = 4 * 1000 * 1000 # 4MB, per MAX_PROTOCOL_MESSAGE_LENGTH -VALID_DATA_LIMIT = MSG_LIMIT - 5 # Account for the 5-byte length prefix +VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" @@ -53,11 +55,13 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_checksum() self.test_size() self.test_msgtype() - self.test_large_inv() + self.test_oversized_inv_msg() + self.test_oversized_getdata_msg() + self.test_oversized_headers_msg() self.test_resource_exhaustion() def test_buffer(self): - self.log.info("Test message with header split across two buffers, should be received") + self.log.info("Test message with header split across two buffers is received") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) # Create valid message msg = conn.build_message(msg_ping(nonce=12345)) @@ -76,6 +80,7 @@ class InvalidMessagesTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() def test_magic_bytes(self): + self.log.info("Test message with invalid magic bytes disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART badmsg']): msg = conn.build_message(msg_unrecognized(str_data="d")) @@ -83,9 +88,10 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = b'\xff' * 4 + msg[4:] conn.send_raw_message(msg) conn.wait_for_disconnect(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_checksum(self): + self.log.info("Test message with invalid checksum logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['CHECKSUM ERROR (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): msg = conn.build_message(msg_unrecognized(str_data="d")) @@ -93,21 +99,22 @@ class InvalidMessagesTest(BitcoinTestFramework): cut_len = 4 + 12 + 4 # modify checksum msg = msg[:cut_len] + b'\xff' * 4 + msg[cut_len + 4:] - self.nodes[0].p2p.send_raw_message(msg) + conn.send_raw_message(msg) conn.sync_with_ping(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_size(self): + self.log.info("Test message with oversized payload disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['']): - # Create a message with oversized payload msg = msg_unrecognized(str_data="d" * (VALID_DATA_LIMIT + 1)) msg = conn.build_message(msg) - self.nodes[0].p2p.send_raw_message(msg) + conn.send_raw_message(msg) conn.wait_for_disconnect(timeout=1) - self.nodes[0].disconnect_p2ps() + self.nodes[0].disconnect_p2ps() def test_msgtype(self): + self.log.info("Test message with invalid message type logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): msg = msg_unrecognized(str_data="d") @@ -115,44 +122,52 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg) # Modify msgtype msg = msg[:7] + b'\x00' + msg[7 + 1:] - self.nodes[0].p2p.send_raw_message(msg) + conn.send_raw_message(msg) conn.sync_with_ping(timeout=1) - self.nodes[0].disconnect_p2ps() - - def test_large_inv(self): - conn = self.nodes[0].add_p2p_connection(P2PInterface()) - with self.nodes[0].assert_debug_log(['Misbehaving', '(0 -> 20): message inv size() = 50001']): - msg = msg_inv([CInv(MSG_TX, 1)] * 50001) - conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['Misbehaving', '(20 -> 40): message getdata size() = 50001']): - msg = msg_getdata([CInv(MSG_TX, 1)] * 50001) - conn.send_and_ping(msg) - with self.nodes[0].assert_debug_log(['Misbehaving', '(40 -> 60): headers message size = 2001']): - msg = msg_headers([CBlockHeader()] * 2001) - conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() + def test_oversized_msg(self, msg, size): + msg_type = msg.msgtype.decode('ascii') + self.log.info("Test {} message of size {} is logged as misbehaving".format(msg_type, size)) + with self.nodes[0].assert_debug_log(['Misbehaving', '{} message size = {}'.format(msg_type, size)]): + self.nodes[0].add_p2p_connection(P2PInterface()).send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + + def test_oversized_inv_msg(self): + size = MAX_INV_SIZE + 1 + self.test_oversized_msg(msg_inv([CInv(MSG_TX, 1)] * size), size) + + def test_oversized_getdata_msg(self): + size = MAX_INV_SIZE + 1 + self.test_oversized_msg(msg_getdata([CInv(MSG_TX, 1)] * size), size) + + def test_oversized_headers_msg(self): + size = MAX_HEADERS_RESULTS + 1 + self.test_oversized_msg(msg_headers([CBlockHeader()] * size), size) + def test_resource_exhaustion(self): + self.log.info("Test node stays up despite many large junk messages") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) conn2 = self.nodes[0].add_p2p_connection(P2PDataStore()) msg_at_size = msg_unrecognized(str_data="b" * VALID_DATA_LIMIT) - assert len(msg_at_size.serialize()) == MSG_LIMIT - - self.log.info("Sending a bunch of large, junk messages to test memory exhaustion. May take a bit...") + assert len(msg_at_size.serialize()) == MAX_PROTOCOL_MESSAGE_LENGTH - # Run a bunch of times to test for memory exhaustion. + self.log.info("(a) Send 80 messages, each of maximum valid data size (4MB)") for _ in range(80): conn.send_message(msg_at_size) # Check that, even though the node is being hammered by nonsense from one # connection, it can still service other peers in a timely way. + self.log.info("(b) Check node still services peers in a timely way") for _ in range(20): conn2.sync_with_ping(timeout=2) - # Peer 1, despite being served up a bunch of nonsense, should still be connected. - self.log.info("Waiting for node to drop junk messages.") + self.log.info("(c) Wait for node to drop junk messages, while remaining connected") conn.sync_with_ping(timeout=400) + + # Despite being served up a bunch of nonsense, the peers should still be connected. assert conn.is_connected + assert conn2.is_connected self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8803086213..25dd765442 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1898,8 +1898,7 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - self.stop_node(2) - self.start_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) + self.restart_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) connect_nodes(self.nodes[0], 2) # We reconnect more than 100 blocks, give it plenty of time diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 6273c229ae..7c70f30ca3 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -241,6 +241,17 @@ class BlockchainTest(BitcoinTestFramework): del res['disk_size'], res3['disk_size'] assert_equal(res, res3) + self.log.info("Test hash_type option for gettxoutsetinfo()") + # Adding hash_type 'hash_serialized_2', which is the default, should + # not change the result. + res4 = node.gettxoutsetinfo(hash_type='hash_serialized_2') + del res4['disk_size'] + assert_equal(res, res4) + + # hash_type none should not return a UTXO set hash. + res5 = node.gettxoutsetinfo(hash_type='none') + assert 'hash_serialized_2' not in res5 + def _test_getblockheader(self): node = self.nodes[0] diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 58d8c4abe1..ca26152e7e 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -46,10 +46,12 @@ class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [["-minrelaytxfee=0.00001000"],["-minrelaytxfee=0.00000500"]] + self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]] self.supports_cli = False def run_test(self): + self.log.info('Get out of IBD for the minfeefilter test') + self.nodes[0].generate(1) self.log.info('Connect nodes both way') connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 660953be9b..e5e62fd646 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -38,6 +38,7 @@ class PSBTTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + # TODO: Re-enable this test with segwit v1 def test_utxo_conversion(self): mining_node = self.nodes[2] offline_node = self.nodes[0] @@ -156,6 +157,10 @@ class PSBTTest(BitcoinTestFramework): # spend single key from node 1 rawtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(rawtx) + # Make sure it has both types of UTXOs + decoded = self.nodes[1].decodepsbt(walletprocesspsbt_out['psbt']) + assert 'non_witness_utxo' in decoded['inputs'][0] + assert 'witness_utxo' in decoded['inputs'][0] assert_equal(walletprocesspsbt_out['complete'], True) self.nodes[1].sendrawtransaction(self.nodes[1].finalizepsbt(walletprocesspsbt_out['psbt'])['hex']) @@ -352,7 +357,8 @@ class PSBTTest(BitcoinTestFramework): for i, signer in enumerate(signers): self.nodes[2].unloadwallet("wallet{}".format(i)) - self.test_utxo_conversion() + # TODO: Re-enable this for segwit v1 + # self.test_utxo_conversion() # Test that psbts with p2pkh outputs are created properly p2pkh = self.nodes[0].getnewaddress(address_type='legacy') diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 05308931e3..81eb881234 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -115,6 +115,8 @@ class AuthServiceProxy(): except OSError as e: retry = ( '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) + # Workaround for a bug on macOS. See https://bugs.python.org/issue33450 + retry = retry or ('[Errno 41] Protocol wrong type for socket' in str(e)) if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4d1dd4422e..eb1244035f 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -45,6 +45,10 @@ MAX_MONEY = 21000000 * COIN BIP125_SEQUENCE_NUMBER = 0xfffffffd # Sequence number that is BIP 125 opt-in and BIP 68-opt-out +MAX_PROTOCOL_MESSAGE_LENGTH = 4000000 # Maximum length of incoming protocol messages +MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result +MAX_INV_SIZE = 50000 # Maximum number of entries in an 'inv' protocol message + NODE_NETWORK = (1 << 0) NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index b6c37bc7e0..e6da33763d 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -26,6 +26,7 @@ import threading from test_framework.messages import ( CBlockHeader, + MAX_HEADERS_RESULTS, MIN_VERSION_SUPPORTED, msg_addr, msg_block, @@ -553,7 +554,6 @@ class P2PDataStore(P2PInterface): return headers_list = [self.block_store[self.last_block_hash]] - maxheaders = 2000 while headers_list[-1].sha256 not in locator.vHave: # Walk back through the block store, adding headers to headers_list # as we go. @@ -569,7 +569,7 @@ class P2PDataStore(P2PInterface): break # Truncate the list if there are too many headers - headers_list = headers_list[:-maxheaders - 1:-1] + headers_list = headers_list[:-MAX_HEADERS_RESULTS - 1:-1] response = msg_headers(headers_list) if response is not None: diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 41f9bde183..2fa48006f4 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -396,11 +396,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= args = args or [] # Warn if bitcoind is already running - # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: + # pgrep exits with code zero when one or more matching processes found + if subprocess.run(["pgrep", "-x", "bitcoind"], stdout=subprocess.DEVNULL).returncode == 0: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) - except (OSError, subprocess.SubprocessError): + except OSError: + # pgrep not supported pass # Warn if there is a cache directory diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 524e1593ba..18f0beb598 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -71,8 +71,7 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') self.assert_raises_tool_error( - 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' - .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + 'Error loading wallet.dat. Is wallet being used by another process?', '-wallet=wallet.dat', 'info', ) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 90d17a806c..8837e13005 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -95,8 +95,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) assert self.nodes[0].getmempoolinfo()['loaded'] # Verify txs no longer in either node's mempool @@ -123,8 +122,7 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.00001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.00001"]) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -145,8 +143,7 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Remove using high relay fee again - self.stop_node(0) - self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + self.restart_node(0, extra_args=["-minrelaytxfee=0.0001"]) assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 780cce9d02..eddd938847 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -110,9 +110,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_equal(self.nodes[0].getwalletinfo()["avoid_reuse"], False) assert_equal(self.nodes[1].getwalletinfo()["avoid_reuse"], True) - # Stop and restart node 1 - self.stop_node(1) - self.start_node(1) + self.restart_node(1) connect_nodes(self.nodes[0], 1) # Flags should still be node1.avoid_reuse=false, node2.avoid_reuse=true diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 797c903dd3..8962362276 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -219,6 +219,60 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + # Sendmany with explicit fee (BTC/kB) + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendmany, + amounts={ address: 10 }, + estimate_mode='bTc/kB') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendmany, + amounts={ address: 10 }, + conf_target=-1, + estimate_mode='bTc/kB') + fee_per_kb = 0.0002500 + explicit_fee_per_byte = Decimal(fee_per_kb) / 1000 + txid = self.nodes[2].sendmany( + amounts={ address: 10 }, + conf_target=fee_per_kb, + estimate_mode='bTc/kB', + ) + self.nodes[2].generate(1) + self.sync_all(self.nodes[0:3]) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), explicit_fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + assert_equal(self.nodes[2].getbalance(), node_2_bal) + node_0_bal += Decimal('10') + assert_equal(self.nodes[0].getbalance(), node_0_bal) + + # Sendmany with explicit fee (SAT/B) + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendmany, + amounts={ address: 10 }, + estimate_mode='sat/b') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendmany, + amounts={ address: 10 }, + conf_target=-1, + estimate_mode='sat/b') + fee_sat_per_b = 2 + fee_per_kb = fee_sat_per_b / 100000.0 + explicit_fee_per_byte = Decimal(fee_per_kb) / 1000 + txid = self.nodes[2].sendmany( + amounts={ address: 10 }, + conf_target=fee_sat_per_b, + estimate_mode='sAT/b', + ) + self.nodes[2].generate(1) + self.sync_all(self.nodes[0:3]) + balance = self.nodes[2].getbalance() + node_2_bal = self.check_fee_amount(balance, node_2_bal - Decimal('10'), explicit_fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + assert_equal(balance, node_2_bal) + node_0_bal += Decimal('10') + assert_equal(self.nodes[0].getbalance(), node_0_bal) + self.start_node(3, self.nodes[3].extra_args) connect_nodes(self.nodes[0], 3) self.sync_all() @@ -349,6 +403,74 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all(self.nodes[0:3]) + # send with explicit btc/kb fee + self.log.info("test explicit fee (sendtoaddress as btc/kb)") + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + prebalance = self.nodes[2].getbalance() + assert prebalance > 2 + address = self.nodes[1].getnewaddress() + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + estimate_mode='BTc/Kb') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + conf_target=-1, + estimate_mode='btc/kb') + txid = self.nodes[2].sendtoaddress( + address=address, + amount=1.0, + conf_target=0.00002500, + estimate_mode='btc/kb', + ) + tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) + self.sync_all(self.nodes[0:3]) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + postbalance = self.nodes[2].getbalance() + fee = prebalance - postbalance - Decimal('1') + assert_fee_amount(fee, tx_size, Decimal('0.00002500')) + + # send with explicit sat/b fee + self.sync_all(self.nodes[0:3]) + self.log.info("test explicit fee (sendtoaddress as sat/b)") + self.nodes[0].generate(1) + prebalance = self.nodes[2].getbalance() + assert prebalance > 2 + address = self.nodes[1].getnewaddress() + # Throw if no conf_target provided + assert_raises_rpc_error(-8, "Selected estimate_mode requires a fee rate", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + estimate_mode='SAT/b') + # Throw if negative feerate + assert_raises_rpc_error(-3, "Amount out of range", + self.nodes[2].sendtoaddress, + address=address, + amount=1.0, + conf_target=-1, + estimate_mode='SAT/b') + txid = self.nodes[2].sendtoaddress( + address=address, + amount=1.0, + conf_target=2, + estimate_mode='SAT/B', + ) + tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) + self.sync_all(self.nodes[0:3]) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + postbalance = self.nodes[2].getbalance() + fee = prebalance - postbalance - Decimal('1') + assert_fee_amount(fee, tx_size, Decimal('0.00002000')) + # 2. Import address from node2 to node1 self.nodes[1].importaddress(address_to_import) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 27197e3b6d..72c85b8832 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -71,6 +71,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() + test_invalid_parameters(rbf_node, dest_address) test_simple_bumpfee_succeeds(self, "default", rbf_node, peer_node, dest_address) test_simple_bumpfee_succeeds(self, "fee_rate", rbf_node, peer_node, dest_address) test_feerate_args(self, rbf_node, peer_node, dest_address) @@ -92,6 +93,28 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) +def test_invalid_parameters(node, dest_address): + txid = spend_one_input(node, dest_address) + # invalid estimate mode + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": "moo", + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": 38, + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": { + "foo": "bar", + }, + }) + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": Decimal("3.141592"), + }) + # confTarget and conf_target + assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { + "confTarget": 123, + "conf_target": 456, + }) def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.log.info('Test simple bumpfee: {}'.format(mode)) @@ -127,9 +150,10 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() - assert_raises_rpc_error(-8, "confTarget can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL, "confTarget": 1}) + assert_raises_rpc_error(-8, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL, "confTarget": 1}) assert_raises_rpc_error(-3, "Unexpected key totalFee", rbf_node.bumpfee, rbfid, {"totalFee": NORMAL}) + assert_raises_rpc_error(-8, "conf_target can't be set with fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate.", rbf_node.bumpfee, rbfid, {"fee_rate":0.00001, "confTarget": 1}) # Bumping to just above minrelay should fail to increase total fee enough, at least assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index cc349c7bc5..ba1e494d9a 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -190,8 +190,7 @@ class WalletDumpTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "already exists", lambda: self.nodes[0].dumpwallet(wallet_enc_dump)) # Restart node with new wallet, and test importwallet - self.stop_node(0) - self.start_node(0, ['-wallet=w2']) + self.restart_node(0, ['-wallet=w2']) # Make sure the address is not IsMine before import result = self.nodes[0].getaddressinfo(multisig_addr) diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index c441b75652..3c336623e2 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -103,8 +103,7 @@ class WalletHDTest(BitcoinTestFramework): self.sync_all() # Needs rescan - self.stop_node(1) - self.start_node(1, extra_args=self.extra_args[1] + ['-rescan']) + self.restart_node(1, extra_args=self.extra_args[1] + ['-rescan']) assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) # Try a RPC based rescan @@ -183,8 +182,7 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[0].generate(10) # Restart node 1 with keypool of 3 and a different wallet self.nodes[1].createwallet(wallet_name='origin', blank=True) - self.stop_node(1) - self.start_node(1, extra_args=['-keypool=3', '-wallet=origin']) + self.restart_node(1, extra_args=['-keypool=3', '-wallet=origin']) connect_nodes(self.nodes[0], 1) # sethdseed restoring and seeing txs to addresses out of the keypool diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index ff9ff34185..88beef1034 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -7,19 +7,36 @@ Verify that a bitcoind node can load multiple wallet files """ from decimal import Decimal +from threading import Thread import os import shutil import time +from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_raises_rpc_error, + get_rpc_proxy, ) FEATURE_LATEST = 169900 +got_loading_error = False +def test_load_unload(node, name): + global got_loading_error + for i in range(10): + if got_loading_error: + return + try: + node.loadwallet(name) + node.unloadwallet(name) + except JSONRPCException as e: + if e.error['code'] == -4 and 'Wallet already being loading' in e.error['message']: + got_loading_error = True + return + class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): @@ -212,6 +229,18 @@ class MultiWalletTest(BitcoinTestFramework): w2 = node.get_wallet_rpc(wallet_names[1]) w2.getwalletinfo() + self.log.info("Concurrent wallet loading") + threads = [] + for _ in range(3): + n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) + t = Thread(target=test_load_unload, args=(n, wallet_names[2], )) + t.start() + threads.append(t) + for t in threads: + t.join() + global got_loading_error + assert_equal(got_loading_error, True) + self.log.info("Load remaining wallets") for wallet_name in wallet_names[2:]: loadwallet_name = self.nodes[0].loadwallet(wallet_name) diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index 497a5dd95e..455f1fc5e8 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -77,8 +77,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): assert_equal(conflicted["walletconflicts"][0], conflicting["txid"]) # Node0 wallet is shutdown - self.stop_node(0) - self.start_node(0) + self.restart_node(0) # The block chain re-orgs and the tx is included in a different block self.nodes[1].generate(9) diff --git a/test/functional/wallet_zapwallettxes.py b/test/functional/wallet_zapwallettxes.py index adebff360a..7f1cdbd20b 100755 --- a/test/functional/wallet_zapwallettxes.py +++ b/test/functional/wallet_zapwallettxes.py @@ -49,17 +49,15 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop-start node0. Both confirmed and unconfirmed transactions remain in the wallet. - self.stop_node(0) - self.start_node(0) + # Restart node0. Both confirmed and unconfirmed transactions remain in the wallet. + self.restart_node(0) assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop node0 and restart with zapwallettxes and persistmempool. The unconfirmed + # Restart node0 with zapwallettxes and persistmempool. The unconfirmed # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. - self.stop_node(0) - self.start_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) + self.restart_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) self.nodes[0].syncwithvalidationinterfacequeue() # Flush mempool to wallet @@ -67,10 +65,9 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - # Stop node0 and restart with zapwallettxes, but not persistmempool. + # Restart node0 with zapwallettxes, but not persistmempool. # The unconfirmed transaction is zapped and is no longer in the wallet. - self.stop_node(0) - self.start_node(0, ["-zapwallettxes=2"]) + self.restart_node(0, ["-zapwallettxes=2"]) # tx1 is still be available because it was confirmed assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 1aacc09bcc..d30a8ca231 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -23,7 +23,7 @@ fi # Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it # is undesirable to crash the whole program. See: src/util/check.h # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") if [[ ${OUTPUT} != "" ]]; then echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." echo diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 5404565b94..611bd4a8c4 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -64,7 +64,7 @@ EXPECTED_BOOST_INCLUDES=( boost/preprocessor/cat.hpp boost/preprocessor/stringize.hpp boost/signals2/connection.hpp - boost/signals2/last_value.hpp + boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp boost/thread/condition_variable.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index e2bb403c4d..2e5b801849 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -97,6 +97,7 @@ LOCALE_DEPENDENT_FUNCTIONS=( snprintf sprintf sscanf + std::locale::global std::to_string stod stof diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index decea38c4f..72e8ef7c7d 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -39,7 +39,6 @@ enabled=( E711 # comparison to None should be 'if cond is None:' E714 # test for object identity should be "is not" E721 # do not compare types, use "isinstance()" - E741 # do not use variables named "l", "O", or "I" E742 # do not define classes named "l", "O", or "I" E743 # do not define functions named "l", "O", or "I" E901 # SyntaxError: invalid syntax diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 563e076b35..9a26cd9c02 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -25,7 +25,6 @@ disabled=( disabled_gitian=( SC2094 # Make sure not to read and write the same file in the same pipeline. SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. - SC2230 # which is non-standard. Use builtin 'command -v' instead. ) EXIT_CODE=0 diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index b8fe75c5c5..cb33dd6232 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -19,7 +19,9 @@ race:CConnman::ThreadMessageHandler race:fHaveGenesis race:ProcessNewBlock race:ThreadImport +race:LoadWallet race:WalletBatch::WriteHDChain +race:BerkeleyBatch race:BerkeleyDatabase race:zmq::* race:bitcoin-qt |