diff options
127 files changed, 3106 insertions, 1185 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 8402dc2216..b4850d49c8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,7 +7,6 @@ clone_depth: 5 environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true CLCACHE_SERVER: 1 - PACKAGES: berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent openssl rapidcheck zeromq double-conversion PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.4/Qt5.9.8_x64_static_vs2019.zip' @@ -15,7 +14,7 @@ environment: QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' cache: -- C:\tools\vcpkg\installed +- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt - C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in - C:\Qt5.9.8_x64_static_vs2019 install: @@ -23,25 +22,27 @@ install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: -# 1. If the vcpkg install directory exists assume dependencies are installed and do nothing. To -# force a fresh install of the packages delete the job's appveyor cache. -# 2. Otherwise: +# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file +# will cause the appveyor cache rules to invalidate the directory) +# 2. If the directory is missing: # a. Update the vcpkg source (including port files) and build the vcpkg binary, -# b. Install the required packages. +# b. Install the missing packages. - ps: | - cd c:\tools\vcpkg + $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt + Write-Host "vcpkg list: $env:PACKAGES" if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { + cd c:\tools\vcpkg $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" git pull origin master .\bootstrap-vcpkg.bat + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + cd "$env:APPVEYOR_BUILD_FOLDER" } else { - Write-Host "vcpkg packages already installed (to reinstall purge appveyor job's cache)." + Write-Host "required vcpkg packages already installed." } - .\vcpkg integrate install - cd "$env:APPVEYOR_BUILD_FOLDER" + c:\tools\vcpkg\vcpkg integrate install before_build: - ps: clcache -M 536870912 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: diff --git a/.cirrus.yml b/.cirrus.yml index 517cd93585..f4a3878ed8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,7 @@ task: ccache_cache: folder: "/tmp/ccache_dir" install_script: - - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache + - pkg install -y autoconf automake boost-libs git gmake libevent libtool pkgconf python3 ccache - ./contrib/install_db4.sh $(pwd) - ccache --max-size=${CCACHE_SIZE} configure_script: diff --git a/.travis.yml b/.travis.yml index 51bb7d6e0b..16f3ad2dba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ dist: xenial os: linux language: minimal +arch: amd64 cache: ccache: true directories: @@ -46,7 +47,8 @@ stages: env: global: - CI_RETRY_EXE="travis_retry" - - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." + - CI_WAIT="while sleep 500; do echo .; done" + - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source ./ci/test/00_setup_env.sh - set -o errexit; source ./ci/test/03_before_install.sh @@ -113,33 +115,33 @@ jobs: - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh" + FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh" + FILE_ENV="./ci/test/00_setup_env_native_trusty.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh" + FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" TEST_RUNNER_EXTRA="--exclude feature_block" # Not enough memory on travis machines - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh" + FILE_ENV="./ci/test/00_setup_env_native_asan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh" + FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh" + FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" - stage: test name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' diff --git a/build_msvc/README.md b/build_msvc/README.md index 0c668b2838..704470cac8 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -12,7 +12,7 @@ Quick Start The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. ``` -vcpkg install --triplet x64-windows-static boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent openssl zeromq berkeleydb rapidcheck double-conversion +vcpkg install --triplet x64-windows-static boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent zeromq berkeleydb rapidcheck double-conversion py -3 build_msvc\msvc-autogen.py msbuild /m build_msvc\bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` @@ -33,7 +33,6 @@ The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/d - Boost - DoubleConversion - libevent -- OpenSSL - Qt5 - RapidCheck - ZeroMQ @@ -51,10 +50,10 @@ Building The instructions below use `vcpkg` to install the dependencies. - Install [`vcpkg`](https://github.com/Microsoft/vcpkg). -- Install the required packages (replace x64 with x86 as required): +- Install the required packages (replace x64 with x86 as required). The list of required packages can be found in the `build_msvc\vcpkg-packages.txt` file. The PowerShell command below will work if run from the repository root directory and `vcpkg` is in the path. Alternatively the contents of the packages text file can be pasted in place of the `Get-Content` cmdlet. ``` -PS >.\vcpkg install --triplet x64-windows-static boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent openssl zeromq berkeleydb rapidcheck double-conversion +PS >.\vcpkg install --triplet x64-windows-static $(Get-Content -Path build_msvc\vcpkg-packages.txt).split() ``` - Use Python to generate `*.vcxproj` from Makefile @@ -68,13 +67,13 @@ PS >py -3 msvc-autogen.py - Build with Visual Studio 2017 or msbuild. ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build ``` - Build with Visual Studio 2019 or msbuild. ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v142 /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` AppVeyor diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index aea961ec69..1d05ff6cba 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -9,15 +9,13 @@ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\*_properties.cpp" /> + <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> - <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> - <ClCompile Include="..\..\src\test\util\*.cpp" /> - <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> <ClCompile Include="..\..\src\test\main.cpp" /> - <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> <ClCompile Include="..\..\src\test\util\*.cpp" /> + <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> + <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> diff --git a/build_msvc/vcpkg-packages.txt b/build_msvc/vcpkg-packages.txt new file mode 100644 index 0000000000..d63636259d --- /dev/null +++ b/build_msvc/vcpkg-packages.txt @@ -0,0 +1 @@ +berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent rapidcheck zeromq double-conversion
\ No newline at end of file diff --git a/ci/README.md b/ci/README.md index fb1cd7460b..880e49b459 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,11 +8,21 @@ and numbered according to which stage and lifecycle step it belongs to. ### Running a stage locally +Be aware that the tests will be built and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. + +The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. +While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, +such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run +the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and +testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci +system in a virtual machine with a Linux operating system of your choice. + To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io bash git +sudo apt install docker.io bash ``` To run the default test stage, @@ -26,6 +36,3 @@ To run the test stage with a specific configuration, ``` FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` - -Be aware that the tests will be build and run in-place, so please run at your own risk. -If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8bc50da2c3..2919072fe4 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -21,7 +21,10 @@ echo "Fallback to default values in env (if not yet set)" export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} -export HOST=${HOST:-x86_64-unknown-linux-gnu} +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} @@ -41,7 +44,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_native_asan.sh index 46b870e145..b354940d35 100644 --- a/ci/test/00_setup_env_amd64_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -6,8 +6,7 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 7bdfc2c320..b0405bb762 100644 --- a/ci/test/00_setup_env_amd64_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,11 +6,11 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu -export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" +export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang-8 CXX=clang++-8" +# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64 diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index d5a2ba3111..53348559be 100644 --- a/ci/test/00_setup_env_amd64_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1" export GOAL="install" diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 55820ea835..622ec3cfe1 100644 --- a/ci/test/00_setup_env_amd64_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_native_trusty.sh index 51e98788c7..893c1f6b53 100644 --- a/ci/test/00_setup_env_amd64_trusty.sh +++ b/ci/test/00_setup_env_native_trusty.sh @@ -6,9 +6,8 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export DOCKER_NAME_TAG=ubuntu:14.04 -export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev" +export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export RUN_FUNCTIONAL_TESTS=false export GOAL="install" diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 82ac988c41..f7a16206ab 100644 --- a/ci/test/00_setup_env_amd64_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -6,9 +6,8 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 271ae82e5c..ce41c37b6e 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -25,8 +25,6 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then done export PATH="/usr/local/opt/ccache/libexec:$PATH" - OPENSSL_PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" - export PKG_CONFIG_PATH=$OPENSSL_PKG_CONFIG_PATH:$PKG_CONFIG_PATH ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES @@ -35,12 +33,6 @@ fi mkdir -p "${BASE_SCRATCH_DIR}" mkdir -p "${CCACHE_DIR}" -if [ ! -d ${DIR_QA_ASSETS} ]; then - git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} -fi -export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - -mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" export ASAN_OPTIONS="detect_stack_use_after_return=1" export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" @@ -56,7 +48,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ + --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \ + -w $BASE_BUILD_DIR \ + --env-file /tmp/env \ + $DOCKER_NAME_TAG) DOCKER_EXEC () { docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" @@ -85,6 +83,18 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi +if [ ! -d ${DIR_QA_ASSETS} ]; then + DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + +DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" + +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Create $BASE_BUILD_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR +fi + if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index a0488f0807..3f0bba19ac 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -13,13 +13,13 @@ else DOCKER_EXEC echo \> \$HOME/.bitcoin fi -mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index c4dc22bdd8..34b8477197 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -19,7 +19,9 @@ else fi END_FOLD +# Create folder on host and docker, so that `cd` works mkdir -p build +DOCKER_EXEC mkdir -p build # Temporarily disable errexit, because Travis macOS fails without error message set +o errexit @@ -27,10 +29,12 @@ cd build || (echo "could not enter build directory"; exit 1) set -o errexit BEGIN_FOLD configure -DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD BEGIN_FOLD distdir +# Create folder on host and docker, so that `cd` works +mkdir -p "bitcoin-$HOST" DOCKER_EXEC make distdir VERSION=$HOST END_FOLD @@ -39,7 +43,7 @@ cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1) set -o errexit BEGIN_FOLD configure -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD set -o errtrace diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 048fe06c84..49962bb49b 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -32,7 +32,7 @@ fi if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - bash -c "while sleep 500; do echo .; done" & # Print dots in case the unit tests take a long time to run + bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi @@ -48,7 +48,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN} END_FOLD fi - -set +o errexit -cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) -set -o errexit diff --git a/configure.ac b/configure.ac index 8bde6d9bc4..2025037bd4 100644 --- a/configure.ac +++ b/configure.ac @@ -329,6 +329,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -341,6 +342,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and @@ -553,13 +555,8 @@ case $host in dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - openssl_prefix=`$BREW --prefix openssl 2>/dev/null` bdb_prefix=`$BREW --prefix berkeley-db4 2>/dev/null` qt5_prefix=`$BREW --prefix qt5 2>/dev/null` - if test x$openssl_prefix != x; then - PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" - export PKG_CONFIG_PATH - fi if test x$bdb_prefix != x; then CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" LIBS="$LIBS -L$bdb_prefix/lib" @@ -788,7 +785,7 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) # FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS # check that it fails to build without memcpy, then that it builds with @@ -948,6 +945,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> [ AC_MSG_RESULT(no)] ) +AC_MSG_CHECKING(for sysctl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> + #include <sys/sysctl.h>]], + [[ static const int name[2] = {CTL_KERN, KERN_VERSION}; + #ifdef __linux__ + #error "Don't use sysctl on Linux, it's deprecated even when it works" + #endif + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], @@ -1241,7 +1250,6 @@ if test x$use_pkgconfig = xyes; then m4_ifdef( [PKG_CHECK_MODULES], [ - PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi @@ -1264,8 +1272,6 @@ if test x$use_pkgconfig = xyes; then ] ) else - AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) - AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) @@ -1585,7 +1591,6 @@ AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) -AC_SUBST(CRYPTO_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 570d4906cd..712352d6b7 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -51,10 +51,8 @@ def build(): os.chdir('gitian-builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_call(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_call(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['wget', '-O' 'osslsigncode-2.0.tar.gz' '-N', '-P', 'inputs', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) + subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) if args.linux: diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e0b9f74397..c5b4ac1e83 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "curl" - "g++-aarch64-linux-gnu" @@ -59,11 +59,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -77,7 +78,7 @@ script: | then echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 4cfca403b1..9957fc0514 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -4,7 +4,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "faketime" remotes: @@ -24,7 +24,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index a563bef778..79b1b22cdf 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "ca-certificates" - "curl" @@ -55,11 +55,12 @@ script: | export ZERO_AR_DATE=1 + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -71,7 +72,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 656c6d9b7a..1f5b9d4507 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -4,17 +4,17 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: -# Once osslsigncode supports openssl 1.1, we can change this back to libssl-dev -- "libssl1.0-dev" +- "libssl-dev" - "autoconf" +- "libtool" +- "pkg-config" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" files: -- "osslsigncode-1.7.1.tar.gz" -- "osslsigncode-Backports-to-1.7.1.patch" +- "osslsigncode-2.0.tar.gz" - "bitcoin-win-unsigned.tar.gz" script: | set -e -o pipefail @@ -23,16 +23,15 @@ script: | SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned - echo "f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 osslsigncode-1.7.1.tar.gz" | sha256sum -c - echo "a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 osslsigncode-Backports-to-1.7.1.patch" | sha256sum -c + echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c mkdir -p ${UNSIGNED_DIR} tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - tar xf osslsigncode-1.7.1.tar.gz - cd osslsigncode-1.7.1 - patch -p1 < ${BUILD_DIR}/osslsigncode-Backports-to-1.7.1.patch + tar xf osslsigncode-2.0.tar.gz + cd osslsigncode-2.0 + ./autogen.sh ./configure --without-gsf --without-curl --disable-dependency-tracking make find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1c3d6a8dc8..4259a10e14 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "curl" - "g++" @@ -48,11 +48,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -64,7 +65,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -79,7 +80,7 @@ script: | for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 9222a40b17..ba3036a89f 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -27,6 +27,7 @@ D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy 37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 0f6d993fd2..3c485c1df6 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -6,7 +6,14 @@ # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin # $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ -# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +# --show-leak-kinds=all src/test/test_bitcoin +# +# To create suppressions for found issues, use the --gen-suppressions=all option: +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all --gen-suppressions=all --show-reachable=yes \ +# --error-limit=no src/test/test_bitcoin +# +# Note that suppressions may depend on OS and/or library versions. { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -27,6 +34,14 @@ fun:__log_put_record } { + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io + obj:*/libdb_cxx-*.so +} +{ Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 Memcheck:Leak match-leak-kinds: reachable @@ -41,3 +56,57 @@ ... fun:_ZN7leveldbL14InitDefaultEnvEv } +{ + Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) + Memcheck:Addr16 + fun:__wcsnlen_sse4_1 + fun:wcsnrtombs +} +{ + Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) + Memcheck:Cond + fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE + fun:_ZN5boost10filesystem6detail28directory_iterator_constructERNS0_18directory_iteratorERKNS0_4pathEPNS_6system10error_codeE + obj:*/libboost_filesystem.so.* +} +{ + Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v) + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ +} +{ + Suppress boost still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_M_construct_aux<char*> + fun:_M_construct<char*> + fun:basic_string + fun:path +} +{ + Suppress LogInstance still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_Z11LogInstancev +} +{ + Suppress secp256k1_context_create still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:secp256k1_context_create +} +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index b4333abaf0..eb45e14f6f 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,19 +1,12 @@ package=libevent -$(package)_version=2.1.8-stable +$(package)_version=2.1.11-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz -$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d -$(package)_patches=fix_android_arc4random_addrandom.patch +$(package)_sha256_hash=229393ab2bf0dc94694f21836846b424f3532585bac3468738b7bf752c03901e -ifneq (,$(findstring android,$(host))) - define $(package)_preprocess_cmds - ./autogen.sh && patch -p1 < $($(package)_patch_dir)/fix_android_arc4random_addrandom.patch - endef -else - define $(package)_preprocess_cmds - ./autogen.sh - endef -endif +define $(package)_preprocess_cmds + ./autogen.sh +endef define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk deleted file mode 100644 index e3b3647dd9..0000000000 --- a/depends/packages/openssl.mk +++ /dev/null @@ -1,88 +0,0 @@ -package=openssl -$(package)_version=1.0.1k -$(package)_download_path=https://www.openssl.org/source -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fcebe7a41c -$(package)_patches=0001-Add-OpenSSL-termios-fix-for-musl-libc.patch - -define $(package)_set_vars -$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" -$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl -$(package)_config_opts+=no-camellia -$(package)_config_opts+=no-capieng -$(package)_config_opts+=no-cast -$(package)_config_opts+=no-comp -$(package)_config_opts+=no-dso -$(package)_config_opts+=no-dtls1 -$(package)_config_opts+=no-ec_nistp_64_gcc_128 -$(package)_config_opts+=no-gost -$(package)_config_opts+=no-gmp -$(package)_config_opts+=no-heartbeats -$(package)_config_opts+=no-idea -$(package)_config_opts+=no-jpake -$(package)_config_opts+=no-krb5 -$(package)_config_opts+=no-libunbound -$(package)_config_opts+=no-md2 -$(package)_config_opts+=no-mdc2 -$(package)_config_opts+=no-rc4 -$(package)_config_opts+=no-rc5 -$(package)_config_opts+=no-rdrand -$(package)_config_opts+=no-rfc3779 -$(package)_config_opts+=no-rsax -$(package)_config_opts+=no-sctp -$(package)_config_opts+=no-seed -$(package)_config_opts+=no-sha0 -$(package)_config_opts+=no-shared -$(package)_config_opts+=no-ssl-trace -$(package)_config_opts+=no-ssl2 -$(package)_config_opts+=no-ssl3 -$(package)_config_opts+=no-static_engine -$(package)_config_opts+=no-store -$(package)_config_opts+=no-unit-test -$(package)_config_opts+=no-weak-ssl-ciphers -$(package)_config_opts+=no-whirlpool -$(package)_config_opts+=no-zlib -$(package)_config_opts+=no-zlib-dynamic -$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) -$(package)_config_opts_linux=-fPIC -Wa,--noexecstack -$(package)_config_opts_x86_64_linux=linux-x86_64 -$(package)_config_opts_i686_linux=linux-generic32 -$(package)_config_opts_arm_linux=linux-generic32 -$(package)_config_opts_armv7l_linux=linux-generic32 -$(package)_config_opts_aarch64_linux=linux-generic64 -$(package)_config_opts_mipsel_linux=linux-generic32 -$(package)_config_opts_mips_linux=linux-generic32 -$(package)_config_opts_powerpc_linux=linux-generic32 -$(package)_config_opts_riscv32_linux=linux-generic32 -$(package)_config_opts_riscv64_linux=linux-generic64 -$(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc -$(package)_config_opts_x86_64_mingw32=mingw64 -$(package)_config_opts_i686_mingw32=mingw -$(package)_config_opts_android=-fPIC -$(package)_config_opts_aarch64_android=linux-generic64 -$(package)_config_opts_x86_64_android=linux-generic64 -$(package)_config_opts_armv7a_android=linux-generic32 -$(package)_config_opts_i686_android=linux-generic32 -endef - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch && \ - sed -i.old "/define DATE/d" util/mkbuildinf.pl && \ - sed -i.old "s|engines apps test|engines|" Makefile.org -endef - -define $(package)_config_cmds - ./Configure $($(package)_config_opts) -endef - -define $(package)_build_cmds - $(MAKE) -j1 build_crypto libcrypto.pc libssl.pc openssl.pc -endef - -define $(package)_stage_cmds - $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw -endef - -define $(package)_postprocess_cmds - rm -rf share bin etc -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 09734c7e6f..09f3187ac4 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,4 @@ -packages:=boost openssl libevent +packages:=boost libevent qt_packages = qrencode zlib diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index a0f7cda2c0..6ac6ba6838 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -202,6 +202,8 @@ define $(package)_preprocess_cmds sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ + sed -i.old "s|QMAKE_CC = clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ + sed -i.old "s|QMAKE_CXX = clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf endef diff --git a/depends/patches/libevent/fix_android_arc4random_addrandom.patch b/depends/patches/libevent/fix_android_arc4random_addrandom.patch deleted file mode 100644 index 5bcc64bef6..0000000000 --- a/depends/patches/libevent/fix_android_arc4random_addrandom.patch +++ /dev/null @@ -1,68 +0,0 @@ -From cadae3ab7abf45e61ecae8aac39d97d1f3cbd336 Mon Sep 17 00:00:00 2001 -From: Lawrence Nahum <lawrence@greenaddress.it> -Date: Sun, 3 Dec 2017 22:56:09 +0100 -Subject: [PATCH] fixup - ---- - configure.ac | 1 + - evutil_rand.c | 3 +++ - include/event2/util.h | 4 ++-- - 3 files changed, 6 insertions(+), 2 deletions(-) - -diff --git a/configure.ac b/configure.ac -index 7528d37..3bb2121 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -341,6 +341,7 @@ dnl Checks for library functions. - AC_CHECK_FUNCS([ \ - accept4 \ - arc4random \ -+ arc4random_addrandom \ - arc4random_buf \ - eventfd \ - epoll_create1 \ -diff --git a/evutil_rand.c b/evutil_rand.c -index 046a14b..3f0bf2c 100644 ---- a/evutil_rand.c -+++ b/evutil_rand.c -@@ -191,6 +191,7 @@ evutil_secure_rng_get_bytes(void *buf, size_t n) - { - ev_arc4random_buf(buf, n); - } -+#ifdef HAVE_ARC4RANDOM_ADDRANDOM - - void - evutil_secure_rng_add_bytes(const char *buf, size_t n) -@@ -199,6 +200,8 @@ evutil_secure_rng_add_bytes(const char *buf, size_t n) - n>(size_t)INT_MAX ? INT_MAX : (int)n); - } - -+#endif -+ - void - evutil_free_secure_rng_globals_(void) - { -diff --git a/include/event2/util.h b/include/event2/util.h -index dd4bbb6..a9a169d 100644 ---- a/include/event2/util.h -+++ b/include/event2/util.h -@@ -841,7 +841,7 @@ int evutil_secure_rng_init(void); - */ - EVENT2_EXPORT_SYMBOL - int evutil_secure_rng_set_urandom_device_file(char *fname); -- -+#ifdef HAVE_ARC4RANDOM_ADDRANDOM - /** Seed the random number generator with extra random bytes. - - You should almost never need to call this function; it should be -@@ -858,7 +858,7 @@ int evutil_secure_rng_set_urandom_device_file(char *fname); - */ - EVENT2_EXPORT_SYMBOL - void evutil_secure_rng_add_bytes(const char *dat, size_t datlen); -- -+#endif - #ifdef __cplusplus - } - #endif --- -2.14.3 diff --git a/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch b/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch deleted file mode 100644 index 003099bdc2..0000000000 --- a/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/crypto/ui/ui_openssl.c b/crypto/ui/ui_openssl.c -index a38c758..d99edc2 100644 ---- a/crypto/ui/ui_openssl.c -+++ b/crypto/ui/ui_openssl.c -@@ -190,9 +190,9 @@ - # undef SGTTY - #endif - --#if defined(linux) && !defined(TERMIO) --# undef TERMIOS --# define TERMIO -+#if defined(linux) -+# define TERMIOS -+# undef TERMIO - # undef SGTTY - #endif - diff --git a/doc/README.md b/doc/README.md index d3017de2ab..46b311b251 100644 --- a/doc/README.md +++ b/doc/README.md @@ -84,5 +84,3 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th License --------------------- Distributed under the [MIT software license](/COPYING). -This product includes software developed by the OpenSSL Project for use in the [OpenSSL Toolkit](https://www.openssl.org/). This product includes -cryptographic software written by Eric Young ([eay@cryptsoft.com](mailto:eay@cryptsoft.com)), and UPnP software written by Thomas Bernard. diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index d22b6e8383..4831623504 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -11,7 +11,7 @@ This guide does not contain instructions for building the GUI. You will need the following dependencies, which can be installed as root via pkg: ```shell -pkg install autoconf automake boost-libs git gmake libevent libtool openssl pkgconf +pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf git clone https://github.com/bitcoin/bitcoin.git ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 9942449bf6..e435798e95 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -19,7 +19,7 @@ Then install [Homebrew](https://brew.sh). ## Dependencies ```shell -brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config python qt libevent qrencode +brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode ``` See [dependencies.md](dependencies.md) for a complete overview. diff --git a/doc/build-unix.md b/doc/build-unix.md index d048bdeff5..e799e709fa 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -33,7 +33,6 @@ These dependencies are required: Library | Purpose | Description ------------|------------------|---------------------- - libssl | Crypto | Random Number Generation, Elliptic Curve Cryptography libboost | Utility | Library for threading, data structures, etc libevent | Networking | OS independent asynchronous networking @@ -81,7 +80,7 @@ Build requirements: Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: - sudo apt-get install libssl-dev libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev + sudo apt-get install libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev BerkeleyDB is required for the wallet. @@ -127,7 +126,7 @@ built by default. Build requirements: - sudo dnf install gcc-c++ libtool make autoconf automake openssl-devel libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 + sudo dnf install gcc-c++ libtool make autoconf automake libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 Optional (see `--with-miniupnpc` and `--enable-upnp-default`): diff --git a/doc/dependencies.md b/doc/dependencies.md index dc88626761..4b57ef2e91 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,11 +13,10 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | | HarfBuzz-NG | | | | | | -| libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | +| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | -| OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index e7fd8102a4..5f9c18f2f5 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -35,6 +35,7 @@ Developer Notes - [GUI](#gui) - [Subtrees](#subtrees) - [Scripted diffs](#scripted-diffs) + - [Suggestions and examples](#suggestions-and-examples) - [Release notes](#release-notes) - [RPC interface guidelines](#rpc-interface-guidelines) @@ -889,7 +890,7 @@ Scripted diffs For reformatting and refactoring commits where the changes can be easily automated using a bash script, we use scripted-diff commits. The bash script is included in the commit message and our Travis CI job checks that the result of the script is identical to the commit. This aids reviewers since they can verify that the script -does exactly what it's supposed to do. It is also helpful for rebasing (since the same script can just be re-run +does exactly what it is supposed to do. It is also helpful for rebasing (since the same script can just be re-run on the new master commit). To create a scripted-diff: @@ -910,7 +911,35 @@ For development, it might be more convenient to verify all scripted-diffs in a r test/lint/commit-script-check.sh origin/master..HEAD ``` -Commit [`bb81e173`](https://github.com/bitcoin/bitcoin/commit/bb81e173) is an example of a scripted-diff. +### Suggestions and examples + +If you need to replace in multiple files, prefer `git ls-files` to `find` or globbing, and `git grep` to `grep`, to +avoid changing files that are not under version control. + +For efficient replacement scripts, reduce the selection to the files that potentially need to be modified, so for +example, instead of a blanket `git ls-files src | xargs sed -i s/apple/orange/`, use +`git grep -l apple src | xargs sed -i s/apple/orange/`. + +Also, it is good to keep the selection of files as specific as possible — for example, replace only in directories where +you expect replacements — because it reduces the risk that a rebase of your commit by re-running the script will +introduce accidental changes. + +Some good examples of scripted-diff: + +- [scripted-diff: Rename InitInterfaces to NodeContext](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +uses an elegant script to replace occurences of multiple terms in all source files. + +- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +replaces specific terms in a list of specific source files. + +- [scripted-diff: Replace fprintf with tfm::format](https://github.com/bitcoin/bitcoin/commit/fac03ec43a15ad547161e37e53ea82482cc508f9) +does a global replacement but excludes certain directories. + +To find all previous uses of scripted diffs in the repository, do: + +``` +git log --grep="-BEGIN VERIFY SCRIPT-" +``` Release notes ------------- diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 95c1d24dff..129651d8e9 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -113,7 +113,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index 1957fb736e..f68be21e8d 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -608,7 +608,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index 6b6071d9b7..b4c7698896 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -114,7 +114,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 index 1cb8cdebcd..aadea09a2b 100644 --- a/doc/man/bitcoin-wallet.1 +++ b/doc/man/bitcoin-wallet.1 @@ -61,7 +61,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index b0aff99ca2..211ba10285 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -581,7 +581,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/release-notes-17410.md b/doc/release-notes-17410.md new file mode 100644 index 0000000000..08ed353889 --- /dev/null +++ b/doc/release-notes-17410.md @@ -0,0 +1,5 @@ +Command-line options +-------------------- + +- The `-debug=db` logging category has been renamed to `-debug=walletdb`, to distinguish it from `coindb`. + `-debug=db` has been deprecated and will be removed in the next major release. diff --git a/doc/release-notes-17437.md b/doc/release-notes-17437.md new file mode 100644 index 0000000000..3edfd00a38 --- /dev/null +++ b/doc/release-notes-17437.md @@ -0,0 +1,5 @@ +Low-level RPC Changes +=== + +- The RPC gettransaction, listtransactions and listsinceblock responses now also +includes the height of the block that contains the wallet transaction, if any. diff --git a/doc/release-notes.md b/doc/release-notes.md index a47c8802b0..33a127e0d0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -63,6 +63,12 @@ distribution provides binaries for the RISC-V platform. Notable changes =============== +Build System +------------ + +- OpenSSL is no longer used by Bitcoin Core. The last usage of the library +was removed in #17265. + New RPCs -------- diff --git a/doc/release-notes/release-notes-0.19.0.md b/doc/release-notes/release-notes-0.19.0.md new file mode 100644 index 0000000000..153da68a6e --- /dev/null +++ b/doc/release-notes/release-notes-0.19.0.md @@ -0,0 +1,1086 @@ +Bitcoin Core version 0.19.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.19.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +built using Qt 5.9.x, which doesn't support versions of macOS older than +10.10. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +Users running macOS Catalina may need to "right-click" and then choose "Open" +to open the Bitcoin Core .dmg. This is due to new signing requirements +imposed by Apple, which the Bitcoin Core project does not yet adhere too. + +Notable changes +=============== + +New user documentation +---------------------- + +- [Reduce memory](https://github.com/bitcoin/bitcoin/blob/master/doc/reduce-memory.md) + suggests configuration tweaks for running Bitcoin Core on systems with + limited memory. (#16339) + +New RPCs +-------- + +- `getbalances` returns an object with all balances (`mine`, + `untrusted_pending` and `immature`). Please refer to the RPC help of + `getbalances` for details. The new RPC is intended to replace + `getbalance`, `getunconfirmedbalance`, and the balance fields in + `getwalletinfo`. These old calls and fields may be removed in a + future version. (#15930, #16239) + +- `setwalletflag` sets and unsets wallet flags that enable or disable + features specific to that existing wallet, such as the new + `avoid_reuse` feature documented elsewhere in these release notes. + (#13756) + +- `getblockfilter` gets the BIP158 filter for the specified block. This + RPC is only enabled if block filters have been created using the + `-blockfilterindex` configuration option. (#14121) + +New settings +------------ + +- `-blockfilterindex` enables the creation of BIP158 block filters for + the entire blockchain. Filters will be created in the background and + currently use about 4 GiB of space. Note: this version of Bitcoin + Core does not serve block filters over the P2P network, although the + local user may obtain block filters using the `getblockfilter` RPC. + (#14121) + +Updated settings +---------------- + +- `whitebind` and `whitelist` now accept a list of permissions to + provide peers connecting using the indicated interfaces or IP + addresses. If no permissions are specified with an address or CIDR + network, the implicit default permissions are the same as previous + releases. See the `bitcoind -help` output for these two options for + details about the available permissions. (#16248) + +- Users setting custom `dbcache` values can increase their setting slightly + without using any more real memory. Recent changes reduced the memory use + by about 9% and made chainstate accounting more accurate (it was underestimating + the use of memory before). For example, if you set a value of "450" before, you + may now set a value of "500" to use about the same real amount of memory. (#16957) + + +Updated RPCs +------------ + +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. + +- `sendmany` no longer has a `minconf` argument. This argument was not + well-specified and would lead to RPC errors even when the wallet's + coin selection succeeded. Users who want to influence coin selection + can use the existing `-spendzeroconfchange`, `-limitancestorcount`, + `-limitdescendantcount` and `-walletrejectlongchains` configuration + arguments. (#15596) + +- `getbalance` and `sendtoaddress`, plus the new RPCs `getbalances` and + `createwallet`, now accept an "avoid_reuse" parameter that controls + whether already used addresses should be included in the operation. + Additionally, `sendtoaddress` will avoid partial spends when + `avoid_reuse` is enabled even if this feature is not already enabled + via the `-avoidpartialspends` command line flag because not doing so + would risk using up the "wrong" UTXO for an address reuse case. + (#13756) + +- RPCs which have an `include_watchonly` argument or `includeWatching` option now default to `true` for watch-only + wallets. Affected RPCs are: `getbalance`, `listreceivedbyaddress`, `listreceivedbylabel`, `listtransactions`, + `listsinceblock`, `gettransaction`, `walletcreatefundedpsbt`, and `fundrawtransaction`. (#16383) + +- `listunspent` now returns a "reused" bool for each output if the + wallet flag "avoid_reuse" is enabled. (#13756) + +- `getblockstats` now uses BlockUndo data instead of the transaction + index, making it much faster, no longer dependent on the `-txindex` + configuration option, and functional for all non-pruned blocks. + (#14802) + +- `utxoupdatepsbt` now accepts a `descriptors` parameter that will fill + out input and output scripts and keys when known. P2SH-witness inputs + will be filled in from the UTXO set when a descriptor is provided that + shows they're spending segwit outputs. See the RPC help text for full + details. (#15427) + +- `sendrawtransaction` and `testmempoolaccept` no longer accept a + `allowhighfees` parameter to fail mempool acceptance if the + transaction fee exceeds the value of the configuration option + `-maxtxfee`. Now there is a hardcoded default maximum feerate that + can be changed when calling either RPC using a `maxfeerate` parameter. + (#15620) + +- `getmempoolancestors`, `getmempooldescendants`, `getmempoolentry`, and + `getrawmempool` no longer return a `size` field unless the + configuration option `-deprecatedrpc=size` is used. Instead a new + `vsize` field is returned with the transaction's virtual size + (consistent with other RPCs such as `getrawtransaction`). (#15637) + +- `getwalletinfo` now includes a `scanning` field that is either `false` + (no scanning) or an object with information about the duration and + progress of the wallet's scanning historical blocks for transactions + affecting its balances. (#15730) + +- `gettransaction` now accepts a third (boolean) argument `verbose`. If + set to `true`, a new `decoded` field will be added to the response containing + the decoded transaction. This field is equivalent to RPC `decoderawtransaction`, + or RPC `getrawtransaction` when `verbose` is passed. (#16185, #16866, #16873) + +- `createwallet` accepts a new `passphrase` parameter. If set, this + will create the new wallet encrypted with the given passphrase. If + unset (the default) or set to an empty string, no encryption will be + used. (#16394) + +- `getchaintxstats` RPC now returns the additional key of + `window_final_block_height`. (#16695) + +- `getmempoolentry` now provides a `weight` field containing the + transaction weight as defined in BIP141. (#16647) + +- The `getnetworkinfo` and `getpeerinfo` commands now contain a new field with decoded network service flags. (#16786) + +- `getdescriptorinfo` now returns an additional `checksum` field + containing the checksum for the unmodified descriptor provided by the + user (that is, before the descriptor is normalized for the + `descriptor` field). (#15986) + +- `joinpsbts` now shuffles the order of the inputs and outputs of the resulting + joined PSBT. Previously, inputs and outputs were added in the order PSBTs were + provided. This made it easy to correlate inputs to outputs, representing a + privacy leak. (#16512) + +- `walletcreatefundedpsbt` now signals BIP125 Replace-by-Fee if the + `-walletrbf` configuration option is set to true. (#15911) + +GUI changes +----------- + +- The GUI wallet now provides bech32 addresses by default. The user may change the address type + during invoice generation using a GUI toggle, or the default address + type may be changed with the `-addresstype` configuration option. + (#15711, #16497) + +- In 0.18.0, a `./configure` flag was introduced to allow disabling BIP70 support in the GUI (support was enabled by default). In 0.19.0, this flag is now __disabled__ by default. If you want to compile Bitcoin Core with BIP70 support in the GUI, you can pass `--enable-bip70` to `./configure`. (#15584) + +Deprecated or removed configuration options +------------------------------------------- + +- `-mempoolreplacement` is removed, although default node behavior + remains the same. This option previously allowed the user to prevent + the node from accepting or relaying BIP125 transaction replacements. + This is different from the remaining configuration option + `-walletrbf`. (#16171) + +Deprecated or removed RPCs +-------------------------- + +- `bumpfee` no longer accepts a `totalFee` option unless the + configuration parameter `deprecatedrpc=totalFee` is specified. This + parameter will be fully removed in a subsequent release. (#15996) + +- `bumpfee` has a new `fee_rate` option as a replacement for the deprecated `totalFee`. (#16727) + +- `generate` is now removed after being deprecated in Bitcoin Core 0.18. + Use the `generatetoaddress` RPC instead. (#15492) + +P2P changes +----------- + +- BIP 61 reject messages were deprecated in v0.18. They are now disabled + by default, but can be enabled by setting the `-enablebip61` command + line option. BIP 61 reject messages will be removed entirely in a + future version of Bitcoin Core. (#14054) + +- To eliminate well-known denial-of-service vectors in Bitcoin Core, + especially for nodes with spinning disks, the default value for the + `-peerbloomfilters` configuration option has been changed to false. + This prevents Bitcoin Core from sending the BIP111 NODE_BLOOM service + flag, accepting BIP37 bloom filters, or serving merkle blocks or + transactions matching a bloom filter. Users who still want to provide + bloom filter support may either set the configuration option to true + to re-enable both BIP111 and BIP37 support or enable just BIP37 + support for specific peers using the updated `-whitelist` and + `-whitebind` configuration options described elsewhere in these + release notes. For the near future, lightweight clients using public + BIP111/BIP37 nodes should still be able to connect to older versions + of Bitcoin Core and nodes that have manually enabled BIP37 support, + but developers of such software should consider migrating to either + using specific BIP37 nodes or an alternative transaction filtering + system. (#16152) + +- By default, Bitcoin Core will now make two additional outbound connections that are exclusively used for block-relay. No transactions or addr messages will be processed on these connections. These connections are designed to add little additional memory or bandwidth resource requirements but should make some partitioning attacks more difficult to carry out. (#15759) + +Miscellaneous CLI Changes +------------------------- + +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to + `chain` and now returns the current network name as defined in BIP70 + (main, test, regtest). (#15566) + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` no longer returns a `bip9_softforks` object. + Instead, information has been moved into the `softforks` object and + an additional `type` field describes how Bitcoin Core determines + whether that soft fork is active (e.g. BIP9 or BIP90). See the RPC + help for details. (#16060) + +- `getblocktemplate` no longer returns a `rules` array containing `CSV` + and `segwit` (the BIP9 deployments that are currently in active + state). (#16060) + +- `getrpcinfo` now returns a `logpath` field with the path to + `debug.log`. (#15483) + +Tests +----- + +- The regression test chain enabled by the `-regtest` command line flag + now requires transactions to not violate standard policy by default. + This is the same default used for mainnet and makes it easier to test + mainnet behavior on regtest. Note that the testnet still allows + non-standard txs by default and that the policy can be locally + adjusted with the `-acceptnonstdtxn` command line flag for both test + chains. (#15891) + +Configuration +------------ + +- A setting specified in the default section but not also specified in a + network-specific section (e.g. testnet) will now produce an error + preventing startup instead of just a warning unless the network is + mainnet. This prevents settings intended for mainnet from being + applied to testnet or regtest. (#15629) + +- On platforms supporting `thread_local`, log lines can be prefixed with + the name of the thread that caused the log. To enable this behavior, + use `-logthreadnames=1`. (#15849) + +Network +------- + +- When fetching a transaction announced by multiple peers, previous versions of + Bitcoin Core would sequentially attempt to download the transaction from each + announcing peer until the transaction is received, in the order that those + peers' announcements were received. In this release, the download logic has + changed to randomize the fetch order across peers and to prefer sending + download requests to outbound peers over inbound peers. This fixes an issue + where inbound peers could prevent a node from getting a transaction. + (#14897, #15834) + +- If a Tor hidden service is being used, Bitcoin Core will be bound to + the standard port 8333 even if a different port is configured for + clearnet connections. This prevents leaking node identity through use + of identical non-default port numbers. (#15651) + +Mempool and transaction relay +----------------------------- + +- Allows one extra single-ancestor transaction per package. Previously, + if a transaction in the mempool had 25 descendants, or it and all of + its descendants were over 101,000 vbytes, any newly-received + transaction that was also a descendant would be ignored. Now, one + extra descendant will be allowed provided it is an immediate + descendant (child) and the child's size is 10,000 vbytes or less. + This makes it possible for two-party contract protocols such as + Lightning Network to give each participant an output they can spend + immediately for Child-Pays-For-Parent (CPFP) fee bumping without + allowing one malicious participant to fill the entire package and thus + prevent the other participant from spending their output. (#15681) + +- Transactions with outputs paying v1 to v16 witness versions (future + segwit versions) are now accepted into the mempool, relayed, and + mined. Attempting to spend those outputs remains forbidden by policy + ("non-standard"). When this change has been widely deployed, wallets + and services can accept any valid bech32 Bitcoin address without + concern that transactions paying future segwit versions will become + stuck in an unconfirmed state. (#15846) + +- Legacy transactions (transactions with no segwit inputs) must now be + sent using the legacy encoding format, enforcing the rule specified in + BIP144. (#14039) + +Wallet +------ + +- When in pruned mode, a rescan that was triggered by an `importwallet`, + `importpubkey`, `importaddress`, or `importprivkey` RPC will only fail + when blocks have been pruned. Previously it would fail when `-prune` + has been set. This change allows setting `-prune` to a high value + (e.g. the disk size) without the calls to any of the import RPCs + failing until the first block is pruned. (#15870) + +- When creating a transaction with a fee above `-maxtxfee` (default 0.1 + BTC), the RPC commands `walletcreatefundedpsbt` and + `fundrawtransaction` will now fail instead of rounding down the fee. + Be aware that the `feeRate` argument is specified in BTC per 1,000 + vbytes, not satoshi per vbyte. (#16257) + +- A new wallet flag `avoid_reuse` has been added (default off). When + enabled, a wallet will distinguish between used and unused addresses, + and default to not use the former in coin selection. When setting + this flag on an existing wallet, rescanning the blockchain is required + to correctly mark previously used destinations. Together with "avoid + partial spends" (added in Bitcoin Core v0.17.0), this can eliminate a + serious privacy issue where a malicious user can track spends by + sending small payments to a previously-paid address that would then + be included with unrelated inputs in future payments. (#13756) + +Build system changes +-------------------- + +- Python >=3.5 is now required by all aspects of the project. This + includes the build systems, test framework and linters. The previously + supported minimum (3.4), was EOL in March 2019. (#14954) + +- The minimum supported miniUPnPc API version is set to 10. This keeps + compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` + packages. Please note, on Debian this package is still vulnerable to + [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) + (in jessie only) and + [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) + (both in jessie and in stretch). (#15993) + +0.19.0 change log +================= + +### Consensus +- #16128 Delete error-prone CScript constructor only used with FindAndDelete (instagibbs) +- #16060 Bury bip9 deployments (jnewbery) + +### Policy +- #15557 Enhance `bumpfee` to include inputs when targeting a feerate (instagibbs) +- #15846 Make sending to future native witness outputs standard (sipa) + +### Block and transaction handling +- #15632 Remove ResendWalletTransactions from the Validation Interface (jnewbery) +- #14121 Index for BIP 157 block filters (jimpo) +- #15141 Rewrite DoS interface between validation and net_processing (sdaftuar) +- #15880 utils and libraries: Replace deprecated Boost Filesystem functions (hebasto) +- #15971 validation: Add compile-time checking for negative locking requirement in LimitValidationInterfaceQueue (practicalswift) +- #15999 init: Remove dead code in LoadChainTip (MarcoFalke) +- #16015 validation: Hold cs_main when reading chainActive in RewindBlockIndex (practicalswift) +- #16056 remove unused magic number from consistency check (instagibbs) +- #16171 Remove -mempoolreplacement to prevent needless block prop slowness (TheBlueMatt) +- #15894 Remove duplicated "Error: " prefix in logs (hebasto) +- #14193 validation: Add missing mempool locks (MarcoFalke) +- #15681 Allow one extra single-ancestor transaction per package (TheBlueMatt) +- #15305 [validation] Crash if disconnecting a block fails (sdaftuar) +- #16471 log correct messages when CPFP fails (jnewbery) +- #16433 txmempool: Remove unused default value MemPoolRemovalReason::UNKNOWN (MarcoFalke) +- #13868 Remove unused fScriptChecks parameter from CheckInputs (Empact) +- #16421 Conservatively accept RBF bumps bumping one tx at the package limits (TheBlueMatt) +- #16854 Prevent UpdateTip log message from being broken up (stevenroose) +- #16956 validation: Make GetWitnessCommitmentIndex public (MarcoFalke) +- #16713 Ignore old versionbit activations to avoid 'unknown softforks' warning (jnewbery) +- #17002 chainparams: Bump assumed chain params (MarcoFalke) +- #16849 Fix block index inconsistency in InvalidateBlock() (sdaftuar) + +### P2P protocol and network code +- #15597 Generate log entry when blocks messages are received unexpectedly (pstratem) +- #15654 Remove unused unsanitized user agent string CNode::strSubVer (MarcoFalke) +- #15689 netaddress: Update CNetAddr for ORCHIDv2 (dongcarl) +- #15834 Fix transaction relay bugs introduced in #14897 and expire transactions from peer in-flight map (sdaftuar) +- #15651 torcontrol: Use the default/standard network port for Tor hidden services, even if the internal port is set differently (luke-jr) +- #16188 Document what happens to getdata of unknown type (MarcoFalke) +- #15649 Add ChaCha20Poly1305@Bitcoin AEAD (jonasschnelli) +- #16152 Disable bloom filtering by default (TheBlueMatt) +- #15993 Drop support of the insecure miniUPnPc versions (hebasto) +- #16197 Use mockable time for tx download (MarcoFalke) +- #16248 Make whitebind/whitelist permissions more flexible (NicolasDorier) +- #16618 [Fix] Allow connection of a noban banned peer (NicolasDorier) +- #16631 Restore default whitelistrelay to true (NicolasDorier) +- #15759 Add 2 outbound block-relay-only connections (sdaftuar) +- #15558 Don't query all DNS seeds at once (sipa) +- #16999 0.19 seeds update (laanwj) + +### Wallet +- #15288 Remove wallet -> node global function calls (ryanofsky) +- #15491 Improve log output for errors during load (gwillen) +- #13541 wallet/rpc: sendrawtransaction maxfeerate (kallewoof) +- #15680 Remove resendwallettransactions RPC method (jnewbery) +- #15508 Refactor analyzepsbt for use outside RPC code (gwillen) +- #15747 Remove plethora of Get*Balance (MarcoFalke) +- #15728 Refactor relay transactions (jnewbery) +- #15639 bitcoin-wallet tool: Drop libbitcoin_server.a dependency (ryanofsky) +- #15853 Remove unused import checkpoints.h (MarcoFalke) +- #15780 add cachable amounts for caching credit/debit values (kallewoof) +- #15778 Move maxtxfee from node to wallet (jnewbery) +- #15901 log on rescan completion (andrewtoth) +- #15917 Avoid logging no_such_file_or_directory error (promag) +- #15452 Replace CScriptID and CKeyID in CTxDestination with dedicated types (instagibbs) +- #15870 Only fail rescan when blocks have actually been pruned (MarcoFalke) +- #15006 Add option to create an encrypted wallet (achow101) +- #16001 Give WalletModel::UnlockContext move semantics (sipa) +- #15741 Batch write imported stuff in importmulti (achow101) +- #16144 do not encrypt wallets with disabled private keys (mrwhythat) +- #15024 Allow specific private keys to be derived from descriptor (meshcollider) +- #13756 "avoid_reuse" wallet flag for improved privacy (kallewoof) +- #16226 Move ismine to the wallet module (achow101) +- #16239 wallet/rpc: follow-up clean-up/fixes to avoid_reuse (kallewoof) +- #16286 refactoring: wallet: Fix GCC 7.4.0 warning (hebasto) +- #16257 abort when attempting to fund a transaction above -maxtxfee (Sjors) +- #16237 Have the wallet give out destinations instead of keys (achow101) +- #16322 Fix -maxtxfee check by moving it to CWallet::CreateTransaction (promag) +- #16361 Remove redundant pre-TopUpKeypool check (instagibbs) +- #16244 Move wallet creation out of the createwallet rpc into its own function (achow101) +- #16227 Refactor CWallet's inheritance chain (achow101) +- #16208 Consume ReserveDestination on successful CreateTransaction (instagibbs) +- #16301 Use CWallet::Import* functions in all import* RPCs (achow101) +- #16402 Remove wallet settings from chainparams (MarcoFalke) +- #16415 Get rid of PendingWalletTx class (ryanofsky) +- #15588 Log the actual wallet file version and no longer publicly expose the "version" record (achow101) +- #16399 Improve wallet creation (fjahr) +- #16475 Enumerate walletdb keys (MarcoFalke) +- #15709 Do not add "setting" key as unknown (Bushstar) +- #16451 Remove CMerkleTx (jnewbery) +- #15906 Move min_depth and max_depth to coin control (amitiuttarwar) +- #16502 Drop unused OldKey (promag) +- #16394 Allow createwallet to take empty passwords to make unencrypted wallets (achow101) +- #15911 Use wallet RBF default for walletcreatefundedpsbt (Sjors) +- #16503 Remove p2pEnabled from Chain interface (ariard) +- #16557 restore coinbase and confirmed/conflicted checks in SubmitMemoryPoolAndRelay() (jnewbery) +- #14934 Descriptor expansion cache clarifications (Sjors) +- #16383 rpcwallet: default include_watchonly to true for watchonly wallets (jb55) +- #16542 Return more specific errors about invalid descriptors (achow101) +- #16572 Fix Char as Bool in Wallet (JeremyRubin) +- #16753 extract PubKey from P2PK script with Solver (theStack) +- #16716 Use wallet name instead of pointer on unload/release (promag) +- #16185 gettransaction: add an argument to decode the transaction (darosior) +- #16745 Translate all initErrors in CreateWalletFromFile (MarcoFalke) +- #16792 Assert that the HRP is lowercase in Bech32::Encode (meshcollider) +- #16624 encapsulate transactions state (ariard) +- #16830 Cleanup walletinitinterface.h (hebasto) +- #16796 Fix segfault in CreateWalletFromFile (MarcoFalke) +- #16866 Rename 'decode' argument in gettransaction method to 'verbose' (jnewbery) +- #16727 Explicit feerate for bumpfee (instagibbs) +- #16609 descriptor: fix missed m_script_arg arg renaming in #14934 (fanquake) + +### RPC and other APIs +- #15492 remove deprecated generate method (Sjors) +- #15566 cli: Replace testnet with chain and return network name as per bip70 (fanquake) +- #15564 cli: Remove duplicate wallet fields from -getinfo (fanquake) +- #15642 Remove deprecated rpc warnings (jnewbery) +- #15637 Rename size to vsize in mempool related calls (fanquake) +- #15620 Uncouple non-wallet rpcs from maxTxFee global (MarcoFalke) +- #15616 Clarify decodescript RPCResult doc (MarcoFalke) +- #15669 Fix help text for signtransactionwithXXX (torkelrogstad) +- #15596 Ignore sendmany::minconf as dummy value (MarcoFalke) +- #15755 remove unused var in rawtransaction.cpp (Bushstar) +- #15746 RPCHelpMan: Always name dictionary keys (MarcoFalke) +- #15748 remove dead mining code (jnewbery) +- #15751 Speed up deriveaddresses for large ranges (sipa) +- #15770 Validate maxfeerate with AmountFromValue (promag) +- #15474 rest/rpc: Make mempoolinfo atomic (promag) +- #15463 Speedup getaddressesbylabel (promag) +- #15784 Remove dependency on interfaces::Chain in SignTransaction (ariard) +- #15323 Expose g_is_mempool_loaded via getmempoolinfo (Empact) +- #15932 Serialize in getblock without cs_main (MarcoFalke) +- #15930 Add balances RPC (MarcoFalke) +- #15730 Show scanning details in getwalletinfo (promag) +- #14802 faster getblockstats using BlockUndo data (FelixWeis) +- #14984 Speedup getrawmempool when verbose=true (promag) +- #16071 Hint for importmulti in help output of importpubkey and importaddress (kristapsk) +- #16063 Mention getwalletinfo where a rescan is triggered (promag) +- #16024 deriveaddresses: Correction of descriptor checksum in RPC example (ccapo) +- #16217 getrawtransaction: inform about blockhash argument when lookup fails (darosior) +- #15427 Add support for descriptors to utxoupdatepsbt (sipa) +- #16262 Allow shutdown while in generateblocks (pstratem) +- #15483 Adding a 'logpath' entry to getrpcinfo (darosior) +- #16325 Clarify that block count means height excl genesis (MarcoFalke) +- #16326 add new utxoupdatepsbt arguments to the CRPCCommand and CPRCCvertParam tables (jnewbery) +- #16332 Add logpath description for getrpcinfo (instagibbs) +- #16240 JSONRPCRequest-aware RPCHelpMan (kallewoof) +- #15996 Deprecate totalfee argument in `bumpfee` (instagibbs) +- #16467 sendrawtransaction help privacy note (jonatack) +- #16596 Fix getblocktemplate CLI example (emilengler) +- #15986 Add checksum to getdescriptorinfo (sipa) +- #16647 add weight to getmempoolentry output (fanquake) +- #16695 Add window final block height to getchaintxstats (leto) +- #16798 Refactor rawtransaction_util's SignTransaction to separate prevtx parsing (achow101) +- #16285 Improve scantxoutset response and help message (promag) +- #16725 Don't show addresses or P2PK in decoderawtransaction (NicolasDorier) +- #16787 Human readable network services (darosior) +- #16251 Improve signrawtransaction error reporting (ajtowns) +- #16873 fix regression in gettransaction (jonatack) +- #16512 Shuffle inputs and outputs after joining psbts (achow101) +- #16521 Use the default maxfeerate value as BTC/kB (Remagpie) +- #16817 Fix casing in getblockchaininfo to be inline with other fields (dangershony) +- #17131 fix -rpcclienttimeout 0 option (fjahr) +- #17249 Add missing deque include to fix build (jbeich) + +### GUI +- #15464 Drop unused return values in WalletFrame (promag) +- #15614 Defer removeAndDeleteWallet when no modal widget is active (promag) +- #15711 Generate bech32 addresses by default (MarcoFalke) +- #15829 update request payment button text and tab description (fanquake) +- #15874 Resolve the qt/guiutil <-> qt/optionsmodel CD (251Labs) +- #15371 Uppercase bech32 addresses in qr codes (benthecarman) +- #15928 Move QRImageWidget to its own file-pair (luke-jr) +- #16113 move coin control "OK" to the right hand side of the dialog (fanquake) +- #16090 Add vertical spacer to peer detail widget (JosuGZ) +- #15886 qt, wallet: Revamp SendConfirmationDialog (hebasto) +- #16263 Use qInfo() if no error occurs (hebasto) +- #16153 Add antialiasing to traffic graph widget (JosuGZ) +- #16350 Remove unused guard (hebasto) +- #16106 Sort wallets in open wallet menu (promag) +- #16291 Stop translating PACKAGE_NAME (MarcoFalke) +- #16380 Remove unused bits from the service flags enum (MarcoFalke) +- #16379 Fix autostart filenames on Linux for testnet/regtest (hebasto) +- #16366 init: Use InitError for all errors in bitcoind/qt (MarcoFalke) +- #16436 Do not create payment server if -disablewallet option provided (hebasto) +- #16514 Remove unused RPCConsole::tabFocus (promag) +- #16497 Generate bech32 addresses by default (take 2, fixup) (MarcoFalke) +- #16349 Remove redundant WalletController::addWallet slot (hebasto) +- #16578 Do not pass in command line arguments to QApplication (achow101) +- #16612 Remove menu icons (laanwj) +- #16677 remove unused PlatformStyle::TextColorIcon (fanquake) +- #16694 Ensure transaction send error is always visible (fanquake) +- #14879 Add warning messages to the debug window (hebasto) +- #16708 Replace obsolete functions of QSslSocket (hebasto) +- #16701 Replace functions deprecated in Qt 5.13 (hebasto) +- #16706 Replace deprecated QSignalMapper by lambda expressions (hebasto) +- #16707 Remove obsolete QModelIndex::child() (hebasto) +- #16758 Replace QFontMetrics::width() with TextWidth() (hebasto) +- #16760 Change uninstall icon on Windows (GChuf) +- #16720 Replace objc_msgSend() function calls with the native Objective-C syntax (hebasto) +- #16788 Update transifex slug for 0.19 (laanwj) +- #15450 Create wallet menu option (achow101) +- #16735 Remove unused menu items for Windows and Linux (GChuf) +- #16826 Do additional character escaping for wallet names and address labels (achow101) +- #15529 Add Qt programs to msvc build (updated, no code changes) (sipsorcery) +- #16714 add prune to intro screen with smart default (Sjors) +- #16858 advise users not to switch wallets when opening a BIP70 URI (jameshilliard) +- #16822 Create wallet menu option follow-ups (jonatack) +- #16882 Re-generate translations before 0.19.0 (MarcoFalke) +- #16928 Rename address checkbox back to bech32 (MarcoFalke) +- #16837 Fix {C{,XX},LD}FLAGS pickup (dongcarl) +- #16971 Change default size of intro frame (emilengler) +- #16988 Periodic translations update (laanwj) +- #16852 When BIP70 is disabled, get PaymentRequest merchant using string search (achow101) +- #16952 make sure to update the UI when deleting a transaction (jonasschnelli) +- #17031 Prevent processing duplicate payment requests (promag) +- #17135 Make polling in ClientModel asynchronous (promag) +- #17120 Fix start timer from non QThread (promag) +- #17257 disable font antialiasing for QR image address (fanquake) + +### Build system +- #14954 Require python 3.5 (MarcoFalke) +- #15580 native_protobuf: avoid system zlib (dongcarl) +- #15601 Switch to python3 (take 3) (MarcoFalke) +- #15581 Make less assumptions about build env (dongcarl) +- #14853 latest RapidCheck (fanquake) +- #15446 Improve depends debuggability (dongcarl) +- #13788 Fix --disable-asm for newer assembly checks/code (luke-jr) +- #12051 add missing debian contrib file to tarball (puchu) +- #15919 Remove unused OpenSSL includes to make it more clear where OpenSSL is used (practicalswift) +- #15978 .gitignore: Don't ignore depends patches (dongcarl) +- #15939 gitian: Remove windows 32 bit build (MarcoFalke) +- #15239 scripts and tools: Move non-linux build source tarballs to "bitcoin-binaries/version" directory (hebasto) +- #14047 Add HKDF_HMAC256_L32 and method to negate a private key (jonasschnelli) +- #16051 add patch to common dependencies (fanquake) +- #16049 switch to secure download of all dependencies (Kemu) +- #16059 configure: Fix thread_local detection (dongcarl) +- #16089 add ability to skip building zeromq (fanquake) +- #15844 Purge libtool archives (dongcarl) +- #15461 update to Boost 1.70 (Sjors) +- #16141 remove GZIP export from gitian descriptors (fanquake) +- #16235 Cleaned up and consolidated msbuild files (no code changes) (sipsorcery) +- #16246 MSVC: Fix error in debug mode (Fix #16245) (NicolasDorier) +- #16183 xtrans: Configure flags cleanup (dongcarl) +- #16258 [MSVC]: Create the config.ini as part of bitcoind build (NicolasDorier) +- #16271 remove -Wall from rapidcheck build flags (fanquake) +- #16309 [MSVC] allow user level project customization (NicolasDorier) +- #16308 [MSVC] Copy build output to src/ automatically after build (NicolasDorier) +- #15457 Check std::system for -[alert|block|wallet]notify (Sjors) +- #16344 use #if HAVE_SYSTEM instead of defined(HAVE_SYSTEM) (Sjors) +- #16352 prune dbus from depends (fanquake) +- #16270 expat 2.2.7 (fanquake) +- #16408 Prune X packages (dongcarl) +- #16386 disable unused Qt features (fanquake) +- #16424 Treat -Wswitch as error when --enable-werror (MarcoFalke) +- #16441 remove qt libjpeg check from bitcoin_qt.m4 (fanquake) +- #16434 Specify AM_CPPFLAGS for ZMQ (domob1812) +- #16534 add Qt Creator Makefile.am.user to .gitignore (Bushstar) +- #16573 disable building libsecp256k1 benchmarks (fanquake) +- #16533 disable libxcb extensions (fanquake) +- #16589 Remove unused src/obj-test folder (MarcoFalke) +- #16435 autoconf: Sane `--enable-debug` defaults (dongcarl) +- #16622 echo property tests status during build (jonatack) +- #16611 Remove src/obj directory from repository (laanwj) +- #16371 ignore macOS make deploy artefacts & add them to clean-local (fanquake) +- #16654 build: update RapidCheck Makefile (jonatack) +- #16370 cleanup package configure flags (fanquake) +- #16746 msbuild: Ignore linker warning (sipsorcery) +- #16750 msbuild: adds bench_bitcoin to auto generated project files (sipsorcery) +- #16810 guix: Remove ssp spec file hack (dongcarl) +- #16477 skip deploying plugins we dont use in macdeployqtplus (fanquake) +- #16413 Bump QT to LTS release 5.9.8 (THETCR) +- #15584 disable BIP70 support by default (fanquake) +- #16871 make building protobuf optional in depends (fanquake) +- #16879 remove redundant sed patching (fanquake) +- #16809 zlib: Move toolchain options to configure (dongcarl) +- #15146 Solve SmartOS FD_ZERO build issue (Empact) +- #16870 update boost macros to latest upstream for improved error reporting (fanquake) +- #16982 Factor out qt translations from build system (laanwj) +- #16926 Add OpenSSL termios fix for musl libc (nmarley) +- #16927 Refresh ZeroMQ 4.3.1 patch (nmarley) +- #17005 Qt version appears only if GUI is being built (ch4ot1c) +- #16468 Exclude depends/Makefile in .gitignore (promag) + +### Tests and QA +- #15296 Add script checking for deterministic line coverage in unit tests (practicalswift) +- #15338 ci: Build and run tests once on freebsd (MarcoFalke) +- #15479 Add .style.yapf (MarcoFalke) +- #15534 lint-format-strings: open files sequentially (fix for OS X) (gwillen) +- #15504 fuzz: Link BasicTestingSetup (shared with unit tests) (MarcoFalke) +- #15473 bench: Benchmark mempooltojson (MarcoFalke) +- #15466 Print remaining jobs in test_runner.py (stevenroose) +- #15631 mininode: Clearer error message on invalid magic bytes (MarcoFalke) +- #15255 Remove travis_wait from lint script (gkrizek) +- #15686 make pruning test faster (jnewbery) +- #15533 .style.yapf: Set column_limit=160 (MarcoFalke) +- #15660 Overhaul p2p_compactblocks.py (sdaftuar) +- #15495 Add regtests for HTTP status codes (domob1812) +- #15772 Properly log named args in authproxy (MarcoFalke) +- #15771 Prevent concurrency issues reading .cookie file (promag) +- #15693 travis: Switch to ubuntu keyserver to avoid timeouts (MarcoFalke) +- #15629 init: Throw error when network specific config is ignored (MarcoFalke) +- #15773 Add BitcoinTestFramework::sync_* methods (MarcoFalke) +- #15797 travis: Bump second timeout to 33 minutes, add rationale (MarcoFalke) +- #15788 Unify testing setups for fuzz, bench, and unit tests (MarcoFalke) +- #15352 Reduce noise level in test_bitcoin output (practicalswift) +- #15779 Add wallet_balance benchmark (MarcoFalke) +- #15843 fix outdated include in blockfilter_index_tests (jamesob) +- #15866 Add missing syncwithvalidationinterfacequeue to wallet_import_rescan (MarcoFalke) +- #15697 Make swap_magic_bytes in p2p_invalid_messages atomic (MarcoFalke) +- #15895 Avoid re-reading config.ini unnecessarily (luke-jr) +- #15896 feature_filelock, interface_bitcoin_cli: Use PACKAGE_NAME in messages rather than hardcoding Bitcoin Core (luke-jr) +- #15897 QA/mininode: Send all headers upfront in send_blocks_and_test to avoid sending an unconnected one (luke-jr) +- #15696 test_runner: Move feature_pruning to base tests (MarcoFalke) +- #15869 Add settings merge test to prevent regresssions (ryanofsky) +- #15758 Add further tests to wallet_balance (MarcoFalke) +- #15841 combine_logs: append node stderr and stdout if it exists (MarcoFalke) +- #15949 test_runner: Move pruning back to extended (MarcoFalke) +- #15927 log thread names by default in functional tests (jnewbery) +- #15664 change default Python block serialization to witness (instagibbs) +- #15988 Add test for ArgsManager::GetChainName (ryanofsky) +- #15963 Make random seed logged and settable (jnewbery) +- #15943 Fail if RPC has been added without tests (MarcoFalke) +- #16036 travis: Run all lint scripts even if one fails (scravy) +- #13555 parameterize adjustment period in versionbits_computeblockversion (JBaczuk) +- #16079 wallet_balance.py: Prevent edge cases (stevenroose) +- #16078 replace tx hash with txid in rawtransaction test (LongShao007) +- #16042 Bump MAX_NODES to 12 (MarcoFalke) +- #16124 Limit Python linting to files in the repo (practicalswift) +- #16143 Mark unit test blockfilter_index_initial_sync as non-deterministic (practicalswift) +- #16214 travis: Fix caching issues (MarcoFalke) +- #15982 Make msg_block a witness block (MarcoFalke) +- #16225 Make coins_tests/updatecoins_simulation_test deterministic (practicalswift) +- #16236 fuzz: Log output even if fuzzer failed (MarcoFalke) +- #15520 cirrus: Run extended test feature_pruning (MarcoFalke) +- #16234 Add test for unknown args (MarcoFalke) +- #16207 stop generating lcov coverage when functional tests fail (asood123) +- #16252 Log to debug.log in all unit tests (MarcoFalke) +- #16289 Add missing ECC_Stop() in GUI rpcnestedtests.cpp (jonasschnelli) +- #16278 Remove unused includes (practicalswift) +- #16302 Add missing syncwithvalidationinterfacequeue to wallet_balance test (MarcoFalke) +- #15538 wallet_bumpfee.py: Make sure coin selection produces change (instagibbs) +- #16294 Create at most one testing setup (MarcoFalke) +- #16299 bench: Move generated data to a dedicated translation unit (promag) +- #16329 Add tests for getblockchaininfo.softforks (MarcoFalke) +- #15687 tool wallet test coverage for unexpected writes to wallet (jonatack) +- #16267 bench: Benchmark blocktojson (fanatid) +- #14505 Add linter to make sure single parameter constructors are marked explicit (practicalswift) +- #16338 Disable other targets when enable-fuzz is set (qmma70) +- #16334 rpc_users: Also test rpcauth.py with password (dongcarl) +- #15282 Replace hard-coded hex tx with class in test framework (stevenroose) +- #16390 Add --filter option to test_runner.py (promag) +- #15891 Require standard txs in regtest by default (MarcoFalke) +- #16374 Enable passing wildcard test names to test runner from root (jonatack) +- #16420 Fix race condition in wallet_encryption test (jonasschnelli) +- #16422 remove redundant setup in addrman_tests (zenosage) +- #16438 travis: Print memory and number of cpus (MarcoFalke) +- #16445 Skip flaky p2p_invalid_messages test on macOS (fjahr) +- #16459 Fix race condition in example_test.py (sdaftuar) +- #16464 Ensure we don't generate a too-big block in p2sh sigops test (sdaftuar) +- #16491 fix deprecated log.warn in feature_dbcrash test (jonatack) +- #15134 Switch one of the Travis jobs to an unsigned char environment (-funsigned-char) (practicalswift) +- #16505 Changes verbosity of msbuild from quiet to normal in the appveyor script (sipsorcery) +- #16293 Make test cases separate functions (MarcoFalke) +- #16470 Fail early on disconnect in mininode.wait_for_* (MarcoFalke) +- #16277 Suppress output in test_bitcoin for expected errors (gertjaap) +- #16493 Fix test failures (MarcoFalke) +- #16538 Add missing sync_blocks to feature_pruning (MarcoFalke) +- #16509 Adapt test framework for chains other than "regtest" (MarcoFalke) +- #16363 Add test for BIP30 duplicate tx (MarcoFalke) +- #16535 Explain why -whitelist is used in feature_fee_estimation (MarcoFalke) +- #16554 only include and use OpenSSL where it's actually needed (BIP70) (fanquake) +- #16598 Remove confusing hash256 function in util (elichai) +- #16595 travis: Use extended 90 minute timeout when available (MarcoFalke) +- #16563 Add unit test for AddTimeData (mzumsande) +- #16561 Use colors and dots in test_runner.py output only if standard output is a terminal (practicalswift) +- #16465 Test p2sh-witness and bech32 in wallet_import_rescan (MarcoFalke) +- #16582 Rework ci (Use travis only as fallback env) (MarcoFalke) +- #16633 travis: Fix test_runner.py timeouts (MarcoFalke) +- #16646 Run tests with UPnP disabled (fanquake) +- #16623 ci: Add environment files for all settings (MarcoFalke) +- #16656 fix rpc_setban.py race (jonasschnelli) +- #16570 Make descriptor tests deterministic (davereikher) +- #16404 Test ZMQ notification after chain reorg (promag) +- #16726 Avoid common Python default parameter gotcha when mutable dict/list:s are used as default parameter values (practicalswift) +- #16739 ci: Pass down $makejobs to test_runner.py, other improvements (MarcoFalke) +- #16767 Check for codespell in lint-spelling.sh (kristapsk) +- #16768 Make lint-includes.sh work from any directory (kristapsk) +- #15257 Scripts and tools: Bump flake8 to 3.7.8 (Empact) +- #16804 Remove unused try-block in assert_debug_log (MarcoFalke) +- #16850 `servicesnames` field in `getpeerinfo` and `getnetworkinfo` (darosior) +- #16551 Test that low difficulty chain fork is rejected (MarcoFalke) +- #16737 Establish only one connection between nodes in rpc_invalidateblock (MarcoFalke) +- #16845 Add notes on how to generate data/wallets/high_minversion (MarcoFalke) +- #16888 Bump timeouts in slow running tests (MarcoFalke) +- #16864 Add python bech32 impl round-trip test (instagibbs) +- #16865 add some unit tests for merkle.cpp (soroosh-sdi) +- #14696 Add explicit references to related CVE's in p2p_invalid_block test (lucash-dev) +- #16907 lint: Add DisabledOpcodeTemplates to whitelist (MarcoFalke) +- #16898 Remove connect_nodes_bi (MarcoFalke) +- #16917 Move common function assert_approx() into util.py (fridokus) +- #16921 Add information on how to add Vulture suppressions (practicalswift) +- #16920 Fix extra_args in wallet_import_rescan.py (MarcoFalke) +- #16918 Make PORT_MIN in test runner configurable (MarcoFalke) +- #16941 travis: Disable feature_block in tsan run due to oom (MarcoFalke) +- #16929 follow-up to rpc: default maxfeerate value as BTC/kB (jonatack) +- #16959 ci: Set $host before setting fallback values (MarcoFalke) +- #16961 Remove python dead code linter (laanwj) +- #16931 add unittests for CheckProofOfWork (soroosh-sdi) +- #16991 Fix service flag comparison check in rpc_net test (luke-jr) (laanwj) +- #16987 Correct docstring param name (jbampton) +- #17015 Explain QT_QPA_PLATFORM for gui tests (MarcoFalke) +- #17006 Enable UBSan for Travis fuzzing job (practicalswift) +- #17086 Fix fs_tests for unknown locales (carnhofdaki) +- #15903 appveyor: Write @PACKAGE_NAME@ to config (MarcoFalke) +- #16742 test: add executable flag for wallet_watchonly.py (theStack) +- #16740 qa: Relax so that the subscriber is ready before publishing zmq messages (#16740) + +### Miscellaneous +- #15335 Fix lack of warning of unrecognized section names (AkioNak) +- #15528 contrib: Bump gitian descriptors for 0.19 (MarcoFalke) +- #15609 scripts and tools: Set 'distro' explicitly (hebasto) +- #15519 Add Poly1305 implementation (jonasschnelli) +- #15643 contrib: Gh-merge: include acks in merge commit (MarcoFalke) +- #15838 scripts and tools: Fetch missing review comments in github-merge.py (nkostoulas) +- #15920 lint: Check that all wallet args are hidden (MarcoFalke) +- #15849 Thread names in logs and deadlock debug tools (jamesob) +- #15650 Handle the result of posix_fallocate system call (lucayepa) +- #15766 scripts and tools: Upgrade gitian image before signing (hebasto) +- #15512 Add ChaCha20 encryption option (XOR) (jonasschnelli) +- #15968 Fix portability issue with pthreads (grim-trigger) +- #15970 Utils and libraries: fix static_assert for macro HAVE_THREAD_LOCAL (orientye) +- #15863 scripts and tools: Ensure repos are up-to-date in gitian-build.py (hebasto) +- #15224 Add RNG strengthening (10ms once every minute) (sipa) +- #15840 Contrib scripts: Filter IPv6 by ASN (abitfan) +- #13998 Scripts and tools: gitian-build.py improvements and corrections (hebasto) +- #15236 scripts and tools: Make --setup command independent (hebasto) +- #16114 contrib: Add curl as a required program in gitian-build.py (fanquake) +- #16046 util: Add type safe gettime (MarcoFalke) +- #15703 Update secp256k1 subtree to latest upstream (sipa) +- #16086 contrib: Use newer config.guess & config.sub in install_db4.sh (fanquake) +- #16130 Don't GPG sign intermediate commits with github-merge tool (stevenroose) +- #16162 scripts: Add key for michael ford (fanquake) to trusted keys list (fanquake) +- #16201 devtools: Always use unabbreviated commit IDs in github-merge.py (laanwj) +- #16112 util: Log early messages (MarcoFalke) +- #16223 devtools: Fetch and display ACKs at sign-off time in github-merge (laanwj) +- #16300 util: Explain why the path is cached (MarcoFalke) +- #16314 scripts and tools: Update copyright_header.py script (hebasto) +- #16158 Fix logic of memory_cleanse() on MSVC and clean up docs (real-or-random) +- #14734 fix an undefined behavior in uint::SetHex (kazcw) +- #16327 scripts and tools: Update ShellCheck linter (hebasto) +- #15277 contrib: Enable building in guix containers (dongcarl) +- #16362 Add bilingual_str type (hebasto) +- #16481 logs: add missing space (harding) +- #16581 sipsorcery gitian key (sipsorcery) +- #16566 util: Refactor upper/lowercase functions (kallewoof) +- #16620 util: Move resolveerrmsg to util/error (MarcoFalke) +- #16625 scripts: Remove github-merge.py (fanquake) +- #15864 Fix datadir handling (hebasto) +- #16670 util: Add join helper to join a list of strings (MarcoFalke) +- #16665 scripts: Move update-translations.py to maintainer-tools repo (fanquake) +- #16730 Support serialization of `std::vector<bool>` (sipa) +- #16556 Fix systemd service file configuration directory setup (setpill) +- #15615 Add log output during initial header sync (jonasschnelli) +- #16774 Avoid unnecessary "Synchronizing blockheaders" log messages (jonasschnelli) +- #16489 log: harmonize bitcoind logging (jonatack) +- #16577 util: Cbufferedfile fixes and unit test (LarryRuane) +- #16984 util: Make thread names shorter (hebasto) +- #17038 Don't rename main thread at process level (laanwj) +- #17184 util: Filter out macos process serial number (hebasto) +- #17095 util: Filter control characters out of log messages (laanwj) +- #17085 init: Change fallback locale to C.UTF-8 (laanwj) +- #16957 9% less memory: make SaltedOutpointHasher noexcept (martinus) + +### Documentation +- #15514 Update Transifex links (fanquake) +- #15513 add "sections" info to example bitcoin.conf (fanquake) +- #15530 Move wallet lock annotations to header (MarcoFalke) +- #15562 remove duplicate clone step in build-windows.md (fanquake) +- #15565 remove release note fragments (fanquake) +- #15444 Additional productivity tips (Sjors) +- #15577 Enable TLS in link to chris.beams.io (JeremyRand) +- #15604 release note for disabling reject messages by default (jnewbery) +- #15611 Add Gitian key for droark (droark) +- #15626 Update ACK description in CONTRIBUTING.md (jonatack) +- #15603 Add more tips to productivity.md (gwillen) +- #15683 Comment for seemingly duplicate LIBBITCOIN_SERVER (Bushstar) +- #15685 rpc-mining: Clarify error messages (MarcoFalke) +- #15760 Clarify sendrawtransaction::maxfeerate==0 help (MarcoFalke) +- #15659 fix findFork comment (r8921039) +- #15718 Improve netaddress comments (dongcarl) +- #15833 remove out-of-date comment on pay-to-witness support (r8921039) +- #15821 Remove upgrade note in release notes from EOL versions (MarcoFalke) +- #15267 explain AcceptToMemoryPoolWorker's coins_to_uncache (jamesob) +- #15887 Align code example style with clang-format (hebasto) +- #15877 Fix -dustrelayfee= argument docs grammar (keepkeyjon) +- #15908 Align MSVC build options with Linux build ones (hebasto) +- #15941 Add historical release notes for 0.18.0 (laanwj) +- #15794 Clarify PR guidelines w/re documentation (dongcarl) +- #15607 Release process updates (jonatack) +- #14364 Clarify -blocksdir usage (sangaman) +- #15777 Add doxygen comments for keypool classes (jnewbery) +- #15820 Add productivity notes for dummy rebases (dongcarl) +- #15922 Explain how to pass in non-fundamental types into functions (MarcoFalke) +- #16080 build/doc: update bitcoin_config.h packages, release process (jonatack) +- #16047 analyzepsbt description in doc/psbt.md (jonatack) +- #16039 add release note for 14954 (fanquake) +- #16139 Add riscv64 to outputs list in release-process.md (JeremyRand) +- #16140 create security policy (narula) +- #16164 update release process for SECURITY.md (jonatack) +- #16213 Remove explicit mention of versions from SECURITY.md (MarcoFalke) +- #16186 doc/lint: Fix spelling errors identified by codespell 1.15.0 (Empact) +- #16149 Rework section on ACK in CONTRIBUTING.md (MarcoFalke) +- #16196 Add release notes for 14897 & 15834 (MarcoFalke) +- #16241 add rapidcheck to vcpkg install list (fanquake) +- #16243 Remove travis badge from readme (MarcoFalke) +- #16256 remove orphaned header in developer notes (jonatack) +- #15964 Improve build-osx document formatting (giulio92) +- #16313 Fix broken link in doc/build-osx.md (jonatack) +- #16330 Use placeholder instead of key expiration date (hebasto) +- #16339 add reduce-memory.md (fanquake) +- #16347 Include static members in Doxygen (dongcarl) +- #15824 Improve netbase comments (dongcarl) +- #16430 Update bips 35, 37 and 111 status (MarcoFalke) +- #16455 Remove downgrading warning in release notes, per 0.18 branch (MarcoFalke) +- #16484 update labels in CONTRIBUTING.md (MarcoFalke) +- #16483 update Python command in msvc readme (sipsorcery) +- #16504 Add release note for the deprecated totalFee option of bumpfee (promag) +- #16448 add note on precedence of options in bitcoin.conf (fanquake) +- #16536 Update and extend benchmarking.md (ariard) +- #16530 Fix grammar and punctuation in developer notes (Tech1k) +- #16574 Add historical release notes for 0.18.1 (laanwj) +- #16585 Update Markdown syntax for bdb packages (emilengler) +- #16586 Mention other ways to conserve memory on compilation (MarcoFalke) +- #16605 Add missing contributor to 0.18.1 release notes (meshcollider) +- #16615 Fix typos in COPYRIGHT (gapeman) +- #16626 Fix spelling error chache -> cache (nilswloewen) +- #16587 Improve versionbits.h documentation (ariard) +- #16643 Add ZMQ dependencies to the Fedora build instructions (hebasto) +- #16634 Refer in rpcbind doc to the manpage (MarcoFalke) +- #16555 mention whitelist is inbound, and applies to blocksonly (Sjors) +- #16645 initial RapidCheck property-based testing documentation (jonatack) +- #16691 improve depends prefix documentation (fanquake) +- #16629 Add documentation for the new whitelist permissions (NicolasDorier) +- #16723 Update labels in CONTRIBUTING.md (hebasto) +- #16461 Tidy up shadowing section (promag) +- #16621 add default bitcoin.conf locations (GChuf) +- #16752 Delete stale URL in test README (michaelfolkson) +- #14862 Declare BLOCK_VALID_HEADER reserved (MarcoFalke) +- #16806 Add issue templates for bug and feature request (MarcoFalke) +- #16857 Elaborate need to re-login on Debian-based after usermod for Tor group (clashicly) +- #16863 Add a missing closing parenthesis in the bitcoin-wallet's help (darosior) +- #16757 CChainState return values (MarcoFalke) +- #16847 add comments clarifying how local services are advertised (jamesob) +- #16812 Fix whitespace errs in .md files, bitcoin.conf, and Info.plist.in (ch4ot1c) +- #16885 Update tx-size-small comment with relevant CVE disclosure (instagibbs) +- #16900 Fix doxygen comment for SignTransaction in rpc/rawtransaction_util (MarcoFalke) +- #16914 Update homebrew instruction for doxygen (Sjors) +- #16912 Remove Doxygen intro from src/bitcoind.cpp (ch4ot1c) +- #16960 replace outdated OpenSSL comment in test README (fanquake) +- #16968 Remove MSVC update step from translation process (laanwj) +- #16953 Improve test READMEs (fjahr) +- #16962 Put PR template in comments (laanwj) +- #16397 Clarify includeWatching for fundrawtransaction (stevenroose) +- #15459 add how to calculate blockchain and chainstate size variables to release process (marcoagner) +- #16997 Update bips.md for 0.19 (laanwj) +- #17001 Remove mention of renamed mapBlocksUnlinked (MarcoFalke) +- #17014 Consolidate release notes before 0.19.0 (move-only) (MarcoFalke) +- #17111 update bips.md with buried BIP9 deployments (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 251 +- Aaron Clauson +- Akio Nakamura +- Alistair Mann +- Amiti Uttarwar +- Andrew Chow +- andrewtoth +- Anthony Towns +- Antoine Riard +- Aseem Sood +- Ben Carman +- Ben Woosley +- bpay +- Carl Dong +- Carnhof Daki +- Chris Capobianco +- Chris Moore +- Chuf +- clashic +- clashicly +- Cory Fields +- Daki Carnhof +- Dan Gershony +- Daniel Edgecumbe +- Daniel Kraft +- Daniel McNally +- darosior +- David A. Harding +- David Reikher +- Douglas Roark +- Elichai Turkel +- Emil +- Emil Engler +- ezegom +- Fabian Jahr +- fanquake +- Felix Weis +- Ferdinando M. Ametrano +- fridokus +- gapeman +- GChuf +- Gert-Jaap Glasbergen +- Giulio Lombardo +- Glenn Willen +- Graham Krizek +- Gregory Sanders +- grim-trigger +- gwillen +- Hennadii Stepanov +- Jack Mallers +- James Hilliard +- James O'Beirne +- Jan Beich +- Jeremy Rubin +- JeremyRand +- Jim Posen +- John Bampton +- John Newbery +- Jon Atack +- Jon Layton +- Jonas Schnelli +- Jonathan "Duke" Leto +- João Barbosa +- Joonmo Yang +- Jordan Baczuk +- Jorge Timón +- Josu Goñi +- Julian Fleischer +- Karl-Johan Alm +- Kaz Wesley +- keepkeyjon +- Kirill Fomichev +- Kristaps Kaupe +- Kristian Kramer +- Larry Ruane +- Lenny Maiorani +- LongShao007 +- Luca Venturini +- lucash-dev +- Luke Dashjr +- marcoagner +- MarcoFalke +- marcuswin +- Martin Ankerl +- Martin Zumsande +- Matt Corallo +- MeshCollider +- Michael Folkson +- Miguel Herranz +- Nathan Marley +- Neha Narula +- nicolas.dorier +- Nils Loewen +- nkostoulas +- orient +- Patrick Strateman +- Peter Bushnell +- Peter Wagner +- Pieter Wuille +- practicalswift +- qmma +- r8921039 +- RJ Rybarczyk +- Russell Yanofsky +- Samuel Dobson +- Sebastian Falbesoner +- setpill +- shannon1916 +- Sjors Provoost +- soroosh-sdi +- Steven Roose +- Suhas Daftuar +- tecnovert +- THETCR +- Tim Ruffing +- Tobias Kaderle +- Torkel Rogstad +- Ulrich Kempken +- whythat +- William Casarin +- Wladimir J. van der Laan +- zenosage + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 2c3c4e3869..6d1e7c69e9 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -116,10 +116,8 @@ Ensure gitian-builder is up-to-date: pushd ./gitian-builder mkdir -p inputs - wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch - echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c - wget -P inputs https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz - echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c + wget -O osslsigncode-2.0.tar.gz -P inputs https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz + echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c popd Create the macOS SDK tarball, see the [macOS build instructions](build-osx.md#deterministic-macos-dmg-notes) for details, and copy it into the inputs directory. diff --git a/libbitcoinconsensus.pc.in b/libbitcoinconsensus.pc.in index eb920c47eb..1ceab280bb 100644 --- a/libbitcoinconsensus.pc.in +++ b/libbitcoinconsensus.pc.in @@ -8,4 +8,3 @@ Description: Library for the Bitcoin consensus protocol. Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lbitcoinconsensus Cflags: -I${includedir} -Requires.private: libcrypto diff --git a/src/Makefile.am b/src/Makefile.am index ff4f071a3c..ffb97f26d7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,7 +19,7 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -119,6 +119,7 @@ BITCOIN_CORE_H = \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/cpuid.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ @@ -175,6 +176,7 @@ BITCOIN_CORE_H = \ protocol.h \ psbt.h \ random.h \ + randomenv.h \ reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ @@ -219,6 +221,7 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/settings.h \ util/string.h \ util/threadnames.h \ util/time.h \ @@ -502,6 +505,7 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ + randomenv.cpp \ rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ @@ -513,6 +517,7 @@ libbitcoin_util_a_SOURCES = \ util/system.cpp \ util/moneystr.cpp \ util/rbf.cpp \ + util/settings.cpp \ util/threadnames.cpp \ util/spanparsing.cpp \ util/strencodings.cpp \ @@ -566,7 +571,7 @@ bitcoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -584,7 +589,7 @@ bitcoin_cli_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) -bitcoin_cli_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) +bitcoin_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) # # bitcoin-tx binary # @@ -605,7 +610,7 @@ bitcoin_tx_LDADD = \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) -bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +bitcoin_tx_LDADD += $(BOOST_LIBS) # # bitcoin-wallet binary # @@ -632,7 +637,7 @@ bitcoin_wallet_LDADD = \ $(LIBSECP256K1) \ $(LIBUNIVALUE) -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) +bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c9e4fcc4bc..acd712c8a2 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -76,7 +76,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 13b1470b58..93e60ff832 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -314,7 +314,6 @@ endif qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_bitcoin_qt_LDADD += $(CRYPTO_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 562b393b22..37e1da2a78 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -63,7 +63,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 6d2b546a28..a2c76df91c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -24,6 +24,7 @@ FUZZ_TARGETS = \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ test/fuzz/parse_iso8601 \ + test/fuzz/psbt \ test/fuzz/script \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ @@ -91,7 +92,6 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(EVENT_LIBS) \ - $(CRYPTO_LIBS) \ $(EVENT_PTHREADS_LIBS) # test_bitcoin binary # @@ -152,6 +152,7 @@ BITCOIN_TESTS =\ test/script_standard_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ + test/settings_tests.cpp \ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ @@ -206,7 +207,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_bitcoin_LDADD += $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDCHECK_LIBS) +test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(RAPIDCHECK_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ @@ -292,6 +293,12 @@ test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_SOURCES = $(FUZZ_SUITE) test/fuzz/psbt.cpp +test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index dd4d3e97ac..0a1cb858ef 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -71,7 +71,7 @@ public: consensus.BIP66Height = 363725; // 00000000000000000379eaa19dce8c9b722d46ae6a57c2f1a988119488b50931 consensus.CSVHeight = 419328; // 000000000000000004a1b34462cb8aeebd5799177f7a29cf28f2d1961716b5b5 consensus.SegwitHeight = 481824; // 0000000000000000001c8018d9cb3b742ef25114f27563e3fc4a1902167f9893 - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 483840; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -178,7 +178,7 @@ public: consensus.BIP66Height = 330776; // 000000002104c8c45e99a8853285a3b592602a3ccde2b832481da85e9e4ba182 consensus.CSVHeight = 770112; // 00000000025e930139bac5c6c31a403776da130831ab85be56578f3fa75369bb consensus.SegwitHeight = 834624; // 00000000002b980fcd729daaa248fd9316a5200e9b367f4ff2c42453e84201ca - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 836640; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h new file mode 100644 index 0000000000..0877ad47d3 --- /dev/null +++ b/src/compat/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include <cpuid.h> + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 3257ee7f97..dda7e5230f 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -8,9 +8,10 @@ #include <assert.h> #include <string.h> +#include <compat/cpuid.h> + #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #if defined(USE_ASM) -#include <cpuid.h> namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); @@ -546,18 +547,7 @@ bool SelfTest() { return true; } - #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) -// We can't use cpuid.h's __get_cpuid as it does not support subleafs. -void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} - /** Check whether the OS has enabled AVX registers. */ bool AVXEnabled() { @@ -572,7 +562,7 @@ bool AVXEnabled() std::string SHA256AutoDetect() { std::string ret = "standard"; -#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) +#if defined(USE_ASM) && defined(HAVE_GETCPUID) bool have_sse4 = false; bool have_xsave = false; bool have_avx = false; @@ -589,7 +579,7 @@ std::string SHA256AutoDetect() (void)enabled_avx; uint32_t eax, ebx, ecx, edx; - cpuid(1, 0, eax, ebx, ecx, edx); + GetCPUID(1, 0, eax, ebx, ecx, edx); have_sse4 = (ecx >> 19) & 1; have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; @@ -597,7 +587,7 @@ std::string SHA256AutoDetect() enabled_avx = AVXEnabled(); } if (have_sse4) { - cpuid(7, 0, eax, ebx, ecx, edx); + GetCPUID(7, 0, eax, ebx, ecx, edx); have_avx2 = (ebx >> 5) & 1; have_shani = (ebx >> 29) & 1; } diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index 4118ac1b18..fc7dd1b87e 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -23,6 +23,7 @@ public: CSHA512& Write(const unsigned char* data, size_t len); void Finalize(unsigned char hash[OUTPUT_SIZE]); CSHA512& Reset(); + uint64_t Size() const { return bytes; } }; #endif // BITCOIN_CRYPTO_SHA512_H diff --git a/src/init.cpp b/src/init.cpp index 013f0536b4..60209eb60a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -564,9 +564,7 @@ std::string LicenseInfo() "\n" + "\n" + _("This is experimental software.").translated + "\n" + - strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s").translated, "COPYING", "<https://opensource.org/licenses/MIT>") + "\n" + - "\n" + - strprintf(_("This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit %s and cryptographic software written by Eric Young and UPnP software written by Thomas Bernard.").translated, "<https://www.openssl.org>") + + strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s").translated, "COPYING", "<https://opensource.org/licenses/MIT>") + "\n"; } @@ -1259,6 +1257,11 @@ bool AppInitMain(NodeContext& node) CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, 60000); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(mempool); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 23099a7799..26856a00d3 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -58,12 +58,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } return nullopt; } - int getBlockDepth(const uint256& hash) override - { - const Optional<int> tip_height = getHeight(); - const Optional<int> height = getBlockHeight(hash); - return tip_height && height ? *tip_height - *height + 1 : 0; - } uint256 getBlockHash(int height) override { LockAssertion lock(::cs_main); @@ -182,11 +176,11 @@ public: const CBlockIndex* index, const std::vector<CTransactionRef>& tx_conflicted) override { - m_notifications->BlockConnected(*block, tx_conflicted); + m_notifications->BlockConnected(*block, tx_conflicted, index->nHeight); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->BlockDisconnected(*block); + m_notifications->BlockDisconnected(*block, index->nHeight); } void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { @@ -353,13 +347,11 @@ public: { return MakeUnique<NotificationsHandlerImpl>(*this, notifications); } - void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) override + void waitForNotificationsIfTipChanged(const uint256& old_tip) override { if (!old_tip.IsNull()) { LOCK(::cs_main); if (old_tip == ::ChainActive().Tip()->GetBlockHash()) return; - CBlockIndex* block = LookupBlockIndex(old_tip); - if (block && block->GetAncestor(::ChainActive().Height()) == ::ChainActive().Tip()) return; } SyncWithValidationInterfaceQueue(); } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 82eeba1160..349af152d5 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -76,10 +76,6 @@ public: //! included in the current chain. virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - //! Get block depth. Returns 1 for chain tip, 2 for preceding block, and - //! so on. Returns 0 for a block not included in the current chain. - virtual int getBlockDepth(const uint256& hash) = 0; - //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; @@ -226,8 +222,8 @@ public: virtual ~Notifications() {} virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} - virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} - virtual void BlockDisconnected(const CBlock& block) {} + virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted, int height) {} + virtual void BlockDisconnected(const CBlock& block, int height) {} virtual void UpdatedBlockTip() {} virtual void ChainStateFlushed(const CBlockLocator& locator) {} }; @@ -236,9 +232,8 @@ public: virtual std::unique_ptr<Handler> handleNotifications(Notifications& notifications) = 0; //! Wait for pending notifications to be processed unless block hash points to the current - //! chain tip, or to a possible descendant of the current chain tip that isn't currently - //! connected. - virtual void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) = 0; + //! chain tip. + virtual void waitForNotificationsIfTipChanged(const uint256& old_tip) = 0; //! Register handler for RPC. Command is not copied, so reference //! needs to remain valid until Handler is disconnected. diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index b6ede08b14..701a748e55 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -31,7 +31,7 @@ namespace interfaces { namespace { //! Construct wallet tx struct. -WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, const CWalletTx& wtx) +WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { WalletTx result; result.tx = wtx.tx; @@ -49,7 +49,7 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co wallet.IsMine(result.txout_address.back()) : ISMINE_NO); } - result.credit = wtx.GetCredit(locked_chain, ISMINE_ALL); + result.credit = wtx.GetCredit(ISMINE_ALL); result.debit = wtx.GetDebit(ISMINE_ALL); result.change = wtx.GetChange(); result.time = wtx.GetTxTime(); @@ -63,21 +63,20 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C { WalletTxStatus result; result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); - result.blocks_to_maturity = wtx.GetBlocksToMaturity(locked_chain); - result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); + result.blocks_to_maturity = wtx.GetBlocksToMaturity(); + result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wtx.IsInMainChain(locked_chain); + result.is_in_main_chain = wtx.IsInMainChain(); return result; } //! Construct wallet TxOut struct. -WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, - CWallet& wallet, +WalletTxOut MakeWalletTxOut(CWallet& wallet, const CWalletTx& wtx, int n, int depth) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) @@ -86,7 +85,7 @@ WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, result.txout = wtx.tx->vout[n]; result.time = wtx.GetTxTime(); result.depth_in_main_chain = depth; - result.is_spent = wallet.IsSpent(locked_chain, wtx.GetHash(), n); + result.is_spent = wallet.IsSpent(wtx.GetHash(), n); return result; } @@ -170,12 +169,14 @@ public: bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { LOCK(m_wallet->cs_wallet); - return m_wallet->AddDestData(dest, key, value); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->AddDestData(batch, dest, key, value); } bool eraseDestData(const CTxDestination& dest, const std::string& key) override { LOCK(m_wallet->cs_wallet); - return m_wallet->EraseDestData(dest, key); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->EraseDestData(batch, dest, key); } std::vector<std::string> getDestValues(const std::string& prefix) override { @@ -235,7 +236,7 @@ public: { auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); - return m_wallet->AbandonTransaction(*locked_chain, txid); + return m_wallet->AbandonTransaction(txid); } bool transactionCanBeBumped(const uint256& txid) override { @@ -282,7 +283,7 @@ public: LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } @@ -293,7 +294,7 @@ public: std::vector<WalletTx> result; result.reserve(m_wallet->mapWallet.size()); for (const auto& entry : m_wallet->mapWallet) { - result.emplace_back(MakeWalletTx(*locked_chain, *m_wallet, entry.second)); + result.emplace_back(MakeWalletTx(*m_wallet, entry.second)); } return result; } @@ -338,7 +339,7 @@ public: in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; tx_status = MakeWalletTxStatus(*locked_chain, mi->second); - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } @@ -407,7 +408,7 @@ public: auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*locked_chain, *m_wallet, *coin.tx, coin.i, coin.nDepth)); + MakeWalletTxOut(*m_wallet, *coin.tx, coin.i, coin.nDepth)); } } return result; @@ -422,9 +423,9 @@ public: result.emplace_back(); auto it = m_wallet->mapWallet.find(output.hash); if (it != m_wallet->mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(*locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0) { - result.back() = MakeWalletTxOut(*locked_chain, *m_wallet, it->second, output.n, depth); + result.back() = MakeWalletTxOut(*m_wallet, it->second, output.n, depth); } } } diff --git a/src/key.cpp b/src/key.cpp index 3ba21753a2..10b6668aae 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -83,13 +83,13 @@ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *ou * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are * included. * - * privkey must point to an output buffer of length at least CKey::PRIVATE_KEY_SIZE bytes. + * privkey must point to an output buffer of length at least CKey::SIZE bytes. * privkeylen must initially be set to the size of the privkey buffer. Upon return it * will be set to the number of bytes used in the buffer. * key32 must point to a 32-byte raw private key. */ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *privkey, size_t *privkeylen, const unsigned char *key32, bool compressed) { - assert(*privkeylen >= CKey::PRIVATE_KEY_SIZE); + assert(*privkeylen >= CKey::SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; if (!secp256k1_ec_pubkey_create(ctx, &pubkey, key32)) { @@ -115,11 +115,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = CPubKey::COMPRESSED_PUBLIC_KEY_SIZE; + pubkeylen = CPubKey::COMPRESSED_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::COMPRESSED_PRIVATE_KEY_SIZE); + assert(*privkeylen == CKey::COMPRESSED_SIZE); } else { static const unsigned char begin[] = { 0x30,0x82,0x01,0x13,0x02,0x01,0x01,0x04,0x20 @@ -141,11 +141,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = CPubKey::PUBLIC_KEY_SIZE; + pubkeylen = CPubKey::SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_UNCOMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::PRIVATE_KEY_SIZE); + assert(*privkeylen == CKey::SIZE); } return 1; } @@ -173,8 +173,8 @@ CPrivKey CKey::GetPrivKey() const { CPrivKey privkey; int ret; size_t privkeylen; - privkey.resize(PRIVATE_KEY_SIZE); - privkeylen = PRIVATE_KEY_SIZE; + privkey.resize(SIZE); + privkeylen = SIZE; ret = ec_privkey_export_der(secp256k1_context_sign, privkey.data(), &privkeylen, begin(), fCompressed); assert(ret); privkey.resize(privkeylen); @@ -184,7 +184,7 @@ CPrivKey CKey::GetPrivKey() const { CPubKey CKey::GetPubKey() const { assert(fValid); secp256k1_pubkey pubkey; - size_t clen = CPubKey::PUBLIC_KEY_SIZE; + size_t clen = CPubKey::SIZE; CPubKey result; int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, begin()); assert(ret); @@ -276,7 +276,7 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); - assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(size() == 32); @@ -19,7 +19,7 @@ /** * secure_allocator is defined in allocators.h * CPrivKey is a serialized private key, with all parameters included - * (PRIVATE_KEY_SIZE bytes) + * (SIZE bytes) */ typedef std::vector<unsigned char, secure_allocator<unsigned char> > CPrivKey; @@ -30,15 +30,15 @@ public: /** * secp256k1: */ - static const unsigned int PRIVATE_KEY_SIZE = 279; - static const unsigned int COMPRESSED_PRIVATE_KEY_SIZE = 214; + static const unsigned int SIZE = 279; + static const unsigned int COMPRESSED_SIZE = 214; /** * see www.keylength.com * script supports up to 75 for single byte push */ static_assert( - PRIVATE_KEY_SIZE >= COMPRESSED_PRIVATE_KEY_SIZE, - "COMPRESSED_PRIVATE_KEY_SIZE is larger than PRIVATE_KEY_SIZE"); + SIZE >= COMPRESSED_SIZE, + "COMPRESSED_SIZE is larger than SIZE"); private: //! Whether this private key is valid. We check for correctness when modifying the key diff --git a/src/logging.cpp b/src/logging.cpp index b01177f23f..9f6b5ede12 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -95,7 +95,15 @@ void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) bool BCLog::Logger::EnableCategory(const std::string& str) { BCLog::LogFlags flag; - if (!GetLogCategory(flag, str)) return false; + if (!GetLogCategory(flag, str)) { + if (str == "db") { + // DEPRECATION: Added in 0.20, should start returning an error in 0.21 + LogPrintf("Warning: logging category 'db' is deprecated, use 'walletdb' instead\n"); + EnableCategory(BCLog::WALLETDB); + return true; + } + return false; + } EnableCategory(flag); return true; } @@ -139,7 +147,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::HTTP, "http"}, {BCLog::BENCH, "bench"}, {BCLog::ZMQ, "zmq"}, - {BCLog::DB, "db"}, + {BCLog::WALLETDB, "walletdb"}, {BCLog::RPC, "rpc"}, {BCLog::ESTIMATEFEE, "estimatefee"}, {BCLog::ADDRMAN, "addrman"}, diff --git a/src/logging.h b/src/logging.h index 9ed41c2b98..a2caef51a8 100644 --- a/src/logging.h +++ b/src/logging.h @@ -39,7 +39,7 @@ namespace BCLog { HTTP = (1 << 3), BENCH = (1 << 4), ZMQ = (1 << 5), - DB = (1 << 6), + WALLETDB = (1 << 6), RPC = (1 << 7), ESTIMATEFEE = (1 << 8), ADDRMAN = (1 << 9), diff --git a/src/psbt.h b/src/psbt.h index 6a5c468058..bcff66f7a1 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -171,7 +171,7 @@ struct PSBTInput case PSBT_IN_PARTIAL_SIG: { // Make sure that the key is the size of pubkey + 1 - if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { + if (key.size() != CPubKey::SIZE + 1 && key.size() != CPubKey::COMPRESSED_SIZE + 1) { throw std::ios_base::failure("Size of key was not the expected size for the type partial signature pubkey"); } // Read in the pubkey from key diff --git a/src/pubkey.cpp b/src/pubkey.cpp index d38df716bd..10953adc35 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -196,8 +196,8 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha if (!secp256k1_ecdsa_recover(secp256k1_context_verify, &pubkey, &sig, hash.begin())) { return false; } - unsigned char pub[PUBLIC_KEY_SIZE]; - size_t publen = PUBLIC_KEY_SIZE; + unsigned char pub[SIZE]; + size_t publen = SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -217,8 +217,8 @@ bool CPubKey::Decompress() { if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } - unsigned char pub[PUBLIC_KEY_SIZE]; - size_t publen = PUBLIC_KEY_SIZE; + unsigned char pub[SIZE]; + size_t publen = SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -227,7 +227,7 @@ bool CPubKey::Decompress() { bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert((nChild >> 31) == 0); - assert(size() == COMPRESSED_PUBLIC_KEY_SIZE); + assert(size() == COMPRESSED_SIZE); unsigned char out[64]; BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); @@ -238,8 +238,8 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { return false; } - unsigned char pub[COMPRESSED_PUBLIC_KEY_SIZE]; - size_t publen = COMPRESSED_PUBLIC_KEY_SIZE; + unsigned char pub[COMPRESSED_SIZE]; + size_t publen = COMPRESSED_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); pubkeyChild.Set(pub, pub + publen); return true; @@ -251,8 +251,8 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; memcpy(code+9, chaincode.begin(), 32); - assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); - memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); + memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_SIZE); } void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { diff --git a/src/pubkey.h b/src/pubkey.h index fd815a871b..76f743da66 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -33,17 +33,17 @@ public: /** * secp256k1: */ - static constexpr unsigned int PUBLIC_KEY_SIZE = 65; - static constexpr unsigned int COMPRESSED_PUBLIC_KEY_SIZE = 33; - static constexpr unsigned int SIGNATURE_SIZE = 72; - static constexpr unsigned int COMPACT_SIGNATURE_SIZE = 65; + static constexpr unsigned int SIZE = 65; + static constexpr unsigned int COMPRESSED_SIZE = 33; + static constexpr unsigned int SIGNATURE_SIZE = 72; + static constexpr unsigned int COMPACT_SIGNATURE_SIZE = 65; /** * see www.keylength.com * script supports up to 75 for single byte push */ static_assert( - PUBLIC_KEY_SIZE >= COMPRESSED_PUBLIC_KEY_SIZE, - "COMPRESSED_PUBLIC_KEY_SIZE is larger than PUBLIC_KEY_SIZE"); + SIZE >= COMPRESSED_SIZE, + "COMPRESSED_SIZE is larger than SIZE"); private: @@ -51,15 +51,15 @@ private: * Just store the serialized data. * Its length can very cheaply be computed from the first byte. */ - unsigned char vch[PUBLIC_KEY_SIZE]; + unsigned char vch[SIZE]; //! Compute the length of a pubkey with a given first byte. unsigned int static GetLen(unsigned char chHeader) { if (chHeader == 2 || chHeader == 3) - return COMPRESSED_PUBLIC_KEY_SIZE; + return COMPRESSED_SIZE; if (chHeader == 4 || chHeader == 6 || chHeader == 7) - return PUBLIC_KEY_SIZE; + return SIZE; return 0; } @@ -140,7 +140,7 @@ public: void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); - if (len <= PUBLIC_KEY_SIZE) { + if (len <= SIZE) { s.read((char*)vch, len); } else { // invalid pubkey, skip available data @@ -179,7 +179,7 @@ public: //! Check whether this is a compressed public key. bool IsCompressed() const { - return size() == COMPRESSED_PUBLIC_KEY_SIZE; + return size() == COMPRESSED_SIZE; } /** diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 234d3865ab..676c15ea43 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -430,16 +430,19 @@ int GuiMain(int argc, char* argv[]) BitcoinApplication app(*node); - // Register meta types used for QMetaObject::invokeMethod - qRegisterMetaType< bool* >(); + // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection + qRegisterMetaType<bool*>(); #ifdef ENABLE_WALLET qRegisterMetaType<WalletModel*>(); #endif - // Need to pass name here as CAmount is a typedef (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) - // IMPORTANT if it is no longer a typedef use the normal variant above - qRegisterMetaType< CAmount >("CAmount"); - qRegisterMetaType< std::function<void()> >("std::function<void()>"); + // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) + qRegisterMetaType<CAmount>("CAmount"); + qRegisterMetaType<size_t>("size_t"); + + qRegisterMetaType<std::function<void()>>("std::function<void()>"); qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); + /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 0214356eaa..64f52fcf45 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -63,7 +63,7 @@ <item row="4" column="2"> <widget class="QLineEdit" name="reqLabel"> <property name="toolTip"> - <string>An optional label to associate with the new receiving address.</string> + <string>An optional label to associate with the new receiving address (used by you to identify an invoice). It is also attached to the payment request.</string> </property> </widget> </item> @@ -93,7 +93,7 @@ <item row="6" column="2"> <widget class="QLineEdit" name="reqMessage"> <property name="toolTip"> - <string>An optional message to attach to the payment request, which will be displayed when the request is opened. Note: The message will not be sent with the payment over the Bitcoin network.</string> + <string>An optional message that is attached to the payment request and may be displayed to the sender.</string> </property> </widget> </item> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index fbf76452ac..980de711db 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -139,10 +139,12 @@ void TestGUI(interfaces::Node& node) wallet->LoadWallet(firstRun); { auto spk_man = wallet->GetLegacyScriptPubKeyMan(); + auto locked_chain = wallet->chain().lock(); LOCK(wallet->cs_wallet); AssertLockHeld(spk_man->cs_wallet); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { auto locked_chain = wallet->chain().lock(); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 095c98d26f..220e41b383 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -34,14 +34,6 @@ HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bo ui->setupUi(this); QString version = QString{PACKAGE_NAME} + " " + tr("version") + " " + QString::fromStdString(FormatFullVersion()); - /* On x86 add a bit specifier to the version so that users can distinguish between - * 32 and 64 bit builds. On other architectures, 32/64 bit may be more ambiguous. - */ -#if defined(__x86_64__) - version += " " + tr("(%1-bit)").arg(64); -#elif defined(__i386__ ) - version += " " + tr("(%1-bit)").arg(32); -#endif if (about) { diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index d7f0617315..4b2b475883 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -61,11 +61,6 @@ void WalletFrame::addWallet(WalletModel *walletModel) walletStack->addWidget(walletView); mapWalletViews[walletModel] = walletView; - // Ensure a walletView is able to show the main window - connect(walletView, &WalletView::showNormalIfMinimized, [this]{ - gui->showNormalIfMinimized(); - }); - connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); } diff --git a/src/qt/walletview.h b/src/qt/walletview.h index e29c4c52f5..8d5a301cdb 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -115,8 +115,6 @@ public Q_SLOTS: void requestedSyncWarningInfo(); Q_SIGNALS: - /** Signal that we want to show the main window */ - void showNormalIfMinimized(); /** Fired when a message should be reported to the user */ void message(const QString &title, const QString &message, unsigned int style); /** Encryption status of wallet changed */ diff --git a/src/random.cpp b/src/random.cpp index 48d20d7d72..47d76d8700 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -5,19 +5,22 @@ #include <random.h> +#include <compat/cpuid.h> #include <crypto/sha512.h> #include <support/cleanse.h> #ifdef WIN32 #include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include <logging.h> // for LogPrint() -#include <sync.h> // for WAIT_LOCK +#include <logging.h> // for LogPrintf() +#include <sync.h> // for Mutex #include <util/time.h> // for GetTime() #include <stdlib.h> #include <thread> +#include <randomenv.h> + #include <support/allocators/secure.h> #ifndef WIN32 @@ -40,15 +43,6 @@ #include <sys/sysctl.h> #endif - -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -#include <cpuid.h> -#endif - -#include <openssl/err.h> -#include <openssl/rand.h> -#include <openssl/conf.h> - [[noreturn]] static void RandFailure() { LogPrintf("Failed to read randomness, aborting\n"); @@ -75,7 +69,7 @@ static inline int64_t GetPerformanceCounter() noexcept #endif } -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#ifdef HAVE_GETCPUID static bool g_rdrand_supported = false; static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; @@ -86,15 +80,6 @@ static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND" #ifdef bit_RDSEED static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); #endif -static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ - // We can't use __get_cpuid as it doesn't support subleafs. -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} static void InitHardwareRand() { @@ -263,44 +248,6 @@ static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA51 memory_cleanse(buffer, sizeof(buffer)); } -static void RandAddSeedPerfmon(CSHA512& hasher) -{ -#ifdef WIN32 - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data - - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if (GetTime() < nLastPerfmon + 10 * 60) - return; - nLastPerfmon = GetTime(); - - std::vector<unsigned char> vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially - } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - hasher.Write(vData.data(), nSize); - memory_cleanse(vData.data(), nSize); - } else { - // Performance data is only a best-effort attempt at improving the - // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, - // so we don't call RandFailure(). - // TODO: Add logging when the logger is made functional before global - // constructors have been invoked. - } -#endif -} - #ifndef WIN32 /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. @@ -396,8 +343,6 @@ void GetOSRand(unsigned char *ent32) #endif } -void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); - namespace { class RNGState { @@ -413,31 +358,15 @@ class RNGState { unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; bool m_strongly_seeded GUARDED_BY(m_mutex) = false; - std::unique_ptr<Mutex[]> m_mutex_openssl; public: RNGState() noexcept { InitHardwareRand(); - - // Init OpenSSL library multithreading support - m_mutex_openssl.reset(new Mutex[CRYPTO_num_locks()]); - CRYPTO_set_locking_callback(LockingCallbackOpenSSL); - - // OpenSSL can optionally load a config file which lists optional loadable modules and engines. - // We don't use them so we don't require the config. However some of our libs may call functions - // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing - // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be - // that the config appears to have been loaded and there are no modules/engines available. - OPENSSL_no_config(); } ~RNGState() { - // Securely erase the memory used by the OpenSSL PRNG - RAND_cleanup(); - // Shutdown OpenSSL library multithreading support - CRYPTO_set_locking_callback(nullptr); } /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. @@ -473,8 +402,6 @@ public: memory_cleanse(buf, 64); return ret; } - - Mutex& GetOpenSSLMutex(int i) { return m_mutex_openssl[i]; } }; RNGState& GetRNGState() noexcept @@ -486,17 +413,6 @@ RNGState& GetRNGState() noexcept } } -void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS -{ - RNGState& rng = GetRNGState(); - - if (mode & CRYPTO_LOCK) { - rng.GetOpenSSLMutex(i).lock(); - } else { - rng.GetOpenSSLMutex(i).unlock(); - } -} - /* A note on the use of noexcept in the seeding functions below: * * None of the RNG code should ever throw any exception, with the sole exception @@ -544,10 +460,6 @@ static void SeedSlow(CSHA512& hasher) noexcept GetOSRand(buffer); hasher.Write(buffer, sizeof(buffer)); - // OpenSSL RNG (for now) - RAND_bytes(buffer, sizeof(buffer)); - hasher.Write(buffer, sizeof(buffer)); - // High-precision timestamp. // // Note that we also commit to a timestamp in the Fast seeder, so we indirectly commit to a @@ -556,22 +468,16 @@ static void SeedSlow(CSHA512& hasher) noexcept } /** Extract entropy from rng, strengthen it, and feed it into hasher. */ -static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept { - static std::atomic<int64_t> last_strengthen{0}; - int64_t last_time = last_strengthen.load(); - int64_t current_time = GetTimeMicros(); - if (current_time > last_time + 60000000) { // Only run once a minute - // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. - unsigned char strengthen_seed[32]; - rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); - // Strengthen it for 10ms (100ms on first run), and feed it into hasher. - Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); - last_strengthen = current_time; - } + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedSleep(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -579,17 +485,13 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); - // Sleep for 1ms - MilliSleep(1); - - // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) - SeedTimestamp(hasher); - - // Windows performance monitor data (once every 10 minutes) - RandAddSeedPerfmon(hasher); + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + LogPrintf("Feeding %i bytes of dynamic environment data into RNG\n", hasher.Size() - old_size); - // Strengthen every minute - SeedStrengthen(hasher, rng); + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); } static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept @@ -600,17 +502,22 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept // Everything that the 'slow' seeder includes. SeedSlow(hasher); - // Windows performance monitor data. - RandAddSeedPerfmon(hasher); + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); - // Strengthen - SeedStrengthen(hasher, rng); + // Static environment data + RandAddStaticEnv(hasher); + LogPrintf("Feeding %i bytes of environment data into RNG\n", hasher.Size() - old_size); + + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); } enum class RNGLevel { FAST, //!< Automatically called by GetRandBytes SLOW, //!< Automatically called by GetStrongRandBytes - SLEEP, //!< Called by RandAddSeedSleep() + PERIODIC, //!< Called by RandAddPeriodic() }; static void ProcRand(unsigned char* out, int num, RNGLevel level) @@ -628,8 +535,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) case RNGLevel::SLOW: SeedSlow(hasher); break; - case RNGLevel::SLEEP: - SeedSleep(hasher, rng); + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); break; } @@ -640,19 +547,11 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedStartup(startup_hasher, rng); rng.MixExtract(out, num, std::move(startup_hasher), true); } - - // For anything but the 'fast' level, feed the resulting RNG output (after an additional hashing step) back into OpenSSL. - if (level != RNGLevel::FAST) { - unsigned char buf[64]; - CSHA512().Write(out, num).Finalize(buf); - RAND_add(buf, sizeof(buf), num); - memory_cleanse(buf, 64); - } } void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } +void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } bool g_mock_deterministic_tests{false}; @@ -716,7 +615,7 @@ bool Random_SanityCheck() uint64_t start = GetPerformanceCounter(); /* This does not measure the quality of randomness, but it does test that - * OSRandom() overwrites all 32 bytes of the output given a maximum + * GetOSRand() overwrites all 32 bytes of the output given a maximum * number of tries. */ static const ssize_t MAX_TRIES = 1024; diff --git a/src/random.h b/src/random.h index 9d1f751773..2d8ab085e3 100644 --- a/src/random.h +++ b/src/random.h @@ -35,7 +35,6 @@ * that fast seeding includes, but additionally: * - OS entropy (/dev/urandom, getrandom(), ...). The application will terminate if * this entropy source fails. - * - Bytes from OpenSSL's RNG (which itself may be seeded from various sources) * - Another high-precision timestamp (indirectly committing to a benchmark of all the * previous sources). * These entropy sources are slower, but designed to make sure the RNG state contains @@ -52,7 +51,6 @@ * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. - * - (On Windows) Through OpenSSL, the screen contents. * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and @@ -85,11 +83,11 @@ uint256 GetRandHash() noexcept; void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** - * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. + * Gather entropy from various expensive sources, and feed them to the PRNG state. * * Thread-safe. */ -void RandAddSeedSleep(); +void RandAddPeriodic(); /** * Fast randomness source. This is seeded once with secure random data, but diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000..ec42ddabc3 --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,507 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <randomenv.h> + +#include <clientversion.h> +#include <compat/cpuid.h> +#include <crypto/sha512.h> +#include <support/cleanse.h> +#include <util/time.h> // for GetTime() +#ifdef WIN32 +#include <compat.h> // for Windows API +#endif + +#include <algorithm> +#include <atomic> +#include <chrono> +#include <climits> +#include <thread> +#include <vector> + +#include <stdint.h> +#include <string.h> +#ifndef WIN32 +#include <sys/types.h> // must go before a number of other headers +#include <fcntl.h> +#include <netinet/in.h> +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/utsname.h> +#include <unistd.h> +#endif +#ifdef __MACH__ +#include <mach/clock.h> +#include <mach/mach.h> +#include <mach/mach_time.h> +#endif +#if HAVE_DECL_GETIFADDRS +#include <ifaddrs.h> +#endif +#if HAVE_SYSCTL +#include <sys/sysctl.h> +#if HAVE_VM_VM_PARAM_H +#include <vm/vm_param.h> +#endif +#if HAVE_SYS_RESOURCES_H +#include <sys/resources.h> +#endif +#if HAVE_SYS_VMMETER_H +#include <sys/vmmeter.h> +#endif +#endif +#ifdef __linux__ +#include <sys/auxv.h> +#endif + +//! Necessary on some platforms +extern char** environ; + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes + static std::atomic<std::chrono::seconds> last_perfmon{std::chrono::seconds{0}}; + auto last_time = last_perfmon.load(); + auto current_time = GetTime<std::chrono::seconds>(); + if (current_time < last_time + std::chrono::minutes{10}) return; + last_perfmon = current_time; + + std::vector<unsigned char> vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template<typename T> +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same<typename std::decay<T>::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + +#if HAVE_SYSCTL +template<int... S> +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif + +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max; ++leaf) { + for (uint32_t subleaf = 0;; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate over subleaves for leaf 4, 11, 13 + if (leaf != 4 && leaf != 11 && leaf != 13) break; + if ((leaf == 4 || leaf == 13) && ax == 0) break; + if (leaf == 11 && (cx & 0xFF00) == 0) break; + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else +# ifndef __MACH__ + // On non-MacOS systems, use various clock_gettime() calls. + struct timespec ts = {}; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts; +# endif +# else + // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, + // and clock_get_time for CALENDAR_CLOCK as a replacement for CLOCK_REALTIME. + hasher << mach_absolute_time(); + // From https://gist.github.com/jbenet/1087739 + clock_serv_t cclock; + mach_timespec_t mts = {}; + if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { + hasher << mts; + mach_port_deallocate(mach_task_self(), cclock); + } +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv = {}; + gettimeofday(&tv, nullptr); + hasher << tv; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl<CTL_KERN, KERN_PROC, KERN_PROC_ALL>(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl<CTL_HW, HW_DISKSTATS>(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl<CTL_VM, VM_LOADAVG>(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl<CTL_VM, VM_TOTAL>(hasher); +# endif +# ifdef VM_METER + AddSysctl<CTL_VM, VM_METER>(hasher); +# endif +# endif +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + +#ifdef __linux__ + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // __linux__ + +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); +#endif + + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl<CTL_HW, HW_MACHINE>(hasher); +# endif +# ifdef HW_MODEL + AddSysctl<CTL_HW, HW_MODEL>(hasher); +# endif +# ifdef HW_NCPU + AddSysctl<CTL_HW, HW_NCPU>(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl<CTL_HW, HW_PHYSMEM>(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl<CTL_HW, HW_USERMEM>(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl<CTL_HW, HW_MACHINE_ARCH>(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl<CTL_HW, HW_REALMEM>(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl<CTL_HW, HW_CPU_FREQ>(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl<CTL_HW, HW_BUS_FREQ>(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl<CTL_HW, HW_CACHELINE>(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl<CTL_KERN, KERN_BOOTFILE>(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl<CTL_KERN, KERN_BOOTTIME>(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl<CTL_KERN, KERN_CLOCKRATE>(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl<CTL_KERN, KERN_HOSTID>(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl<CTL_KERN, KERN_HOSTUUID>(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl<CTL_KERN, KERN_HOSTNAME>(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl<CTL_KERN, KERN_OSRELDATE>(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl<CTL_KERN, KERN_OSRELEASE>(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl<CTL_KERN, KERN_OSTYPE>(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl<CTL_KERN, KERN_VERSION>(hasher); +# endif +# endif +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); +#endif + hasher << std::this_thread::get_id(); +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000..46cea6f6f2 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include <crypto/sha512.h> + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif diff --git a/src/scheduler.cpp b/src/scheduler.cpp index fdc859b3a0..07a54335ac 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -41,8 +41,6 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); - // Use this chance to get more entropy - RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 13cdd6c61a..32b388b7fa 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -815,8 +815,8 @@ std::unique_ptr<DescriptorImpl> ParseScript(Span<const char>& sp, ParseScriptCon } } if (ctx == ParseScriptContext::P2SH) { - if (script_size + 3 > 520) { - error = strprintf("P2SH script is too large, %d bytes is larger than 520 bytes", script_size + 3); + if (script_size + 3 > MAX_SCRIPT_ELEMENT_SIZE) { + error = strprintf("P2SH script is too large, %d bytes is larger than %d bytes", script_size + 3, MAX_SCRIPT_ELEMENT_SIZE); return nullptr; } } diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 20fae2eebf..ad833bc025 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -61,17 +61,17 @@ static inline void popstack(std::vector<valtype>& stack) } bool static IsCompressedOrUncompressedPubKey(const valtype &vchPubKey) { - if (vchPubKey.size() < CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() < CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: too short return false; } if (vchPubKey[0] == 0x04) { - if (vchPubKey.size() != CPubKey::PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::SIZE) { // Non-canonical public key: invalid length for uncompressed key return false; } } else if (vchPubKey[0] == 0x02 || vchPubKey[0] == 0x03) { - if (vchPubKey.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: invalid length for compressed key return false; } @@ -83,7 +83,7 @@ bool static IsCompressedOrUncompressedPubKey(const valtype &vchPubKey) { } bool static IsCompressedPubKey(const valtype &vchPubKey) { - if (vchPubKey.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: invalid length for compressed key return false; } diff --git a/src/script/sign.h b/src/script/sign.h index 9d0a5b4d70..4c2403f83f 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -101,7 +101,7 @@ template<typename Stream> void DeserializeHDKeypaths(Stream& s, const std::vector<unsigned char>& key, std::map<CPubKey, KeyOriginInfo>& hd_keypaths) { // Make sure that the key is the size of pubkey + 1 - if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { + if (key.size() != CPubKey::SIZE + 1 && key.size() != CPubKey::COMPRESSED_SIZE + 1) { throw std::ios_base::failure("Size of key was not the expected size for the type BIP32 keypath"); } // Read in the pubkey from key diff --git a/src/script/standard.cpp b/src/script/standard.cpp index fc6898f444..144bdcff98 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -44,12 +44,12 @@ const char* GetTxnOutputType(txnouttype t) static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) { - if (script.size() == CPubKey::PUBLIC_KEY_SIZE + 2 && script[0] == CPubKey::PUBLIC_KEY_SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::PUBLIC_KEY_SIZE + 1); + if (script.size() == CPubKey::SIZE + 2 && script[0] == CPubKey::SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::SIZE + 1); return CPubKey::ValidSize(pubkey); } - if (script.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 2 && script[0] == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1); + if (script.size() == CPubKey::COMPRESSED_SIZE + 2 && script[0] == CPubKey::COMPRESSED_SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_SIZE + 1); return CPubKey::ValidSize(pubkey); } return false; diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 5c2050e4a2..85e3351e72 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -23,6 +23,10 @@ #endif #include <algorithm> +#ifdef ARENA_DEBUG +#include <iomanip> +#include <iostream> +#endif LockedPoolManager* LockedPoolManager::_instance = nullptr; std::once_flag LockedPoolManager::init_flag; @@ -137,7 +141,7 @@ Arena::Stats Arena::stats() const } #ifdef ARENA_DEBUG -static void printchunk(char* base, size_t sz, bool used) { +static void printchunk(void* base, size_t sz, bool used) { std::cout << "0x" << std::hex << std::setw(16) << std::setfill('0') << base << " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz << @@ -149,7 +153,7 @@ void Arena::walk() const printchunk(chunk.first, chunk.second, true); std::cout << std::endl; for (const auto& chunk: chunks_free) - printchunk(chunk.first, chunk.second, false); + printchunk(chunk.first, chunk.second->first, false); std::cout << std::endl; } #endif diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp new file mode 100644 index 0000000000..1ce28f9a6d --- /dev/null +++ b/src/test/fuzz/psbt.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> + +#include <node/psbt.h> +#include <optional.h> +#include <psbt.h> +#include <pubkey.h> +#include <script/script.h> +#include <streams.h> +#include <util/memory.h> +#include <version.h> + +#include <cstdint> +#include <string> +#include <vector> + +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + PartiallySignedTransaction psbt_mut; + const std::string raw_psbt{buffer.begin(), buffer.end()}; + std::string error; + if (!DecodeRawPSBT(psbt_mut, raw_psbt, error)) { + return; + } + const PartiallySignedTransaction psbt = psbt_mut; + + const PSBTAnalysis analysis = AnalyzePSBT(psbt); + (void)PSBTRoleName(analysis.next); + for (const PSBTInputAnalysis& input_analysis : analysis.inputs) { + (void)PSBTRoleName(input_analysis.next); + } + + (void)psbt.IsNull(); + (void)psbt.IsSane(); + + Optional<CMutableTransaction> tx = psbt.tx; + if (tx) { + const CMutableTransaction& mtx = *tx; + const PartiallySignedTransaction psbt_from_tx{mtx}; + } + + for (const PSBTInput& input : psbt.inputs) { + (void)PSBTInputSigned(input); + (void)input.IsNull(); + (void)input.IsSane(); + } + + for (const PSBTOutput& output : psbt.outputs) { + (void)output.IsNull(); + } + + for (size_t i = 0; i < psbt.tx->vin.size(); ++i) { + CTxOut tx_out; + if (psbt.GetInputUTXO(tx_out, i)) { + (void)tx_out.IsNull(); + (void)tx_out.ToString(); + } + } + + psbt_mut = psbt; + (void)FinalizePSBT(psbt_mut); + + psbt_mut = psbt; + CMutableTransaction result; + if (FinalizeAndExtractPSBT(psbt_mut, result)) { + const PartiallySignedTransaction psbt_from_tx{result}; + } + + psbt_mut = psbt; + (void)psbt_mut.Merge(psbt); +} diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 8813921df5..03dce552fc 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -345,8 +345,8 @@ BOOST_AUTO_TEST_CASE(merkle_test_BlockWitness) hashes[0].SetNull(); hashes[1] = block.vtx[1]->GetHash(); - uint256 merkelRootofHashes = ComputeMerkleRoot(hashes); + uint256 merkleRootofHashes = ComputeMerkleRoot(hashes); - BOOST_CHECK_EQUAL(merkelRootofHashes, blockWitness); + BOOST_CHECK_EQUAL(merkleRootofHashes, blockWitness); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp new file mode 100644 index 0000000000..b0ee76ea6b --- /dev/null +++ b/src/test/settings_tests.cpp @@ -0,0 +1,163 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <test/util.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> +#include <univalue.h> +#include <util/strencodings.h> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup) + +//! Check settings struct contents against expected json strings. +static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +{ + util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false); + util::SettingsValue list_value(util::SettingsValue::VARR); + for (const auto& item : GetSettingsList(settings, "section", "name", false)) { + list_value.push_back(item); + } + BOOST_CHECK_EQUAL(single_value.write().c_str(), single_val); + BOOST_CHECK_EQUAL(list_value.write().c_str(), list_val); +}; + +// Simple settings merge test case. +BOOST_AUTO_TEST_CASE(Simple) +{ + util::Settings settings; + settings.command_line_options["name"].push_back("val1"); + settings.command_line_options["name"].push_back("val2"); + settings.ro_config["section"]["name"].push_back(2); + + // The last given arg takes precedence when specified via commandline. + CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); + + util::Settings settings2; + settings2.ro_config["section"]["name"].push_back("val2"); + settings2.ro_config["section"]["name"].push_back("val3"); + + // The first given arg takes precedence when specified via config file. + CheckValues(settings2, R"("val2")", R"(["val2","val3"])"); +} + +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentionally. +struct MergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { END, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ActionList arg_actions = {}; + // command_line_options do not have sections. Only iterate over SET and NEGATE + ForEachNoDup(arg_actions, SET, NEGATE, [&]{ + ActionList conf_actions = {}; + ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&]{ + for (bool force_set : {false, true}) { + for (bool ignore_default_section_config : {false, true}) { + fn(arg_actions, conf_actions, force_set, ignore_default_section_config); + } + } + }); + }); + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see comments below). +BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + const std::string& network = CBaseChainParams::MAIN; + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool force_set, + bool ignore_default_section_config) { + std::string desc; + int value_suffix = 0; + util::Settings settings; + + const std::string& name = ignore_default_section_config ? "wallet" : "server"; + auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, + std::vector<util::SettingsValue>& dest) { + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + dest.push_back(value_prefix + std::to_string(++value_suffix)); + desc += " " + name_prefix + name + "=" + dest.back().get_str(); + } + } else if (action == NEGATE || action == SECTION_NEGATE) { + dest.push_back(false); + desc += " " + name_prefix + "no" + name; + } + }; + + if (force_set) { + settings.forced_settings[name] = "forced"; + desc += " " + name + "=forced"; + } + for (Action arg_action : arg_actions) { + push_values(arg_action, "a", "-", settings.command_line_options[name]); + } + for (Action conf_action : conf_actions) { + bool use_section = conf_action == SECTION_SET || conf_action == SECTION_NEGATE; + push_values(conf_action, "c", use_section ? network + "." : "", + settings.ro_config[use_section ? network : ""][name]); + } + + desc += " || "; + desc += GetSetting(settings, network, name, ignore_default_section_config, /* get_chain_name= */ false).write(); + desc += " |"; + for (const auto& s : GetSettingsList(settings, network, name, ignore_default_section_config)) { + desc += " "; + desc += s.write(); + } + desc += " |"; + if (OnlyHasDefaultSectionSetting(settings, network, name)) desc += " ignored"; + desc += "\n"; + + out_sha.Write((const unsigned char*)desc.data(), desc.size()); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + + // If check below fails, should manually dump the results with: + // + // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=settings_tests/Merge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || GetSetting() | GetSettingsList() | OnlyHasDefaultSectionSetting() + BOOST_CHECK_EQUAL(out_sha_hex, "79db02d74e3e193196541b67c068b40ebd0c124a24b3ecbe9cbf7e85b1c4ba7a"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 2d55554acb..eb0050a4a3 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -784,6 +784,19 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "multi-op-return"); + + // Check large scriptSig (non-standard if size is >1650 bytes) + t.vout.resize(1); + t.vout[0].nValue = MAX_MONEY; + t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + // OP_PUSHDATA2 with len (3 bytes) + data (1647 bytes) = 1650 bytes + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1647, 0); // 1650 + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1648, 0); // 1651 + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptsig-size"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index daf6d951bc..b9fcd97a8f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,6 +17,7 @@ #include <stdint.h> #include <thread> +#include <univalue.h> #include <utility> #include <vector> #ifndef WIN32 @@ -166,14 +167,12 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) struct TestArgsManager : public ArgsManager { TestArgsManager() { m_network_only_args.clear(); } - std::map<std::string, std::vector<std::string> >& GetOverrideArgs() { return m_override_args; } - std::map<std::string, std::vector<std::string> >& GetConfigArgs() { return m_config_args; } void ReadConfigString(const std::string str_config) { std::istringstream streamConfig(str_config); { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } std::string error; @@ -193,6 +192,7 @@ struct TestArgsManager : public ArgsManager using ArgsManager::ReadConfigStream; using ArgsManager::cs_args; using ArgsManager::m_network; + using ArgsManager::m_settings; }; BOOST_AUTO_TEST_CASE(util_ParseParameters) @@ -206,28 +206,29 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, ccc, d}); BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 3 && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 3 && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); - BOOST_CHECK(testArgs.GetOverrideArgs().count("-a") && testArgs.GetOverrideArgs().count("-b") && testArgs.GetOverrideArgs().count("-ccc") - && !testArgs.GetOverrideArgs().count("f") && !testArgs.GetOverrideArgs().count("-d")); - - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].size() == 1); - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].front() == ""); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].size() == 2); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].front() == "argument"); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].back() == "multiple"); + BOOST_CHECK(testArgs.m_settings.command_line_options.count("a") && testArgs.m_settings.command_line_options.count("b") && testArgs.m_settings.command_line_options.count("ccc") + && !testArgs.m_settings.command_line_options.count("f") && !testArgs.m_settings.command_line_options.count("d")); + + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].size() == 1); + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].front().get_str() == ""); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].size() == 2); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].front().get_str() == "argument"); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].back().get_str() == "multiple"); BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } @@ -298,6 +299,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) const char *argv_test[] = { "ignored", "-a", "-nob", "-c=0", "-d=1", "-e=false", "-f=true"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, c, d, e, f}); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); @@ -306,8 +308,8 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) BOOST_CHECK(testArgs.IsArgSet({'-', opt}) || !opt); // Nothing else should be in the map - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 6 && - testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 6 && + testArgs.m_settings.ro_config.empty()); // The -no prefix should get stripped on the way in. BOOST_CHECK(!testArgs.IsArgSet("-nob")); @@ -403,6 +405,7 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) "iii=2\n"; TestArgsManager test_args; + LOCK(test_args.cs_args); const auto a = std::make_pair("-a", ArgsManager::ALLOW_BOOL); const auto b = std::make_pair("-b", ArgsManager::ALLOW_BOOL); const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_STRING); @@ -419,22 +422,25 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) // expectation: a, b, ccc, d, fff, ggg, h, i end up in map // so do sec1.ccc, sec1.d, sec1.h, sec2.ccc, sec2.iii - BOOST_CHECK(test_args.GetOverrideArgs().empty()); - BOOST_CHECK(test_args.GetConfigArgs().size() == 13); - - BOOST_CHECK(test_args.GetConfigArgs().count("-a") - && test_args.GetConfigArgs().count("-b") - && test_args.GetConfigArgs().count("-ccc") - && test_args.GetConfigArgs().count("-d") - && test_args.GetConfigArgs().count("-fff") - && test_args.GetConfigArgs().count("-ggg") - && test_args.GetConfigArgs().count("-h") - && test_args.GetConfigArgs().count("-i") + BOOST_CHECK(test_args.m_settings.command_line_options.empty()); + BOOST_CHECK(test_args.m_settings.ro_config.size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config[""].size() == 8); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); + + BOOST_CHECK(test_args.m_settings.ro_config[""].count("a") + && test_args.m_settings.ro_config[""].count("b") + && test_args.m_settings.ro_config[""].count("ccc") + && test_args.m_settings.ro_config[""].count("d") + && test_args.m_settings.ro_config[""].count("fff") + && test_args.m_settings.ro_config[""].count("ggg") + && test_args.m_settings.ro_config[""].count("h") + && test_args.m_settings.ro_config[""].count("i") ); - BOOST_CHECK(test_args.GetConfigArgs().count("-sec1.ccc") - && test_args.GetConfigArgs().count("-sec1.h") - && test_args.GetConfigArgs().count("-sec2.ccc") - && test_args.GetConfigArgs().count("-sec2.iii") + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc") + && test_args.m_settings.ro_config["sec1"].count("h") + && test_args.m_settings.ro_config["sec2"].count("ccc") + && test_args.m_settings.ro_config["sec2"].count("iii") ); BOOST_CHECK(test_args.IsArgSet("-a") @@ -573,24 +579,25 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) BOOST_AUTO_TEST_CASE(util_GetArg) { TestArgsManager testArgs; - testArgs.GetOverrideArgs().clear(); - testArgs.GetOverrideArgs()["strtest1"] = {"string..."}; + LOCK(testArgs.cs_args); + testArgs.m_settings.command_line_options.clear(); + testArgs.m_settings.command_line_options["strtest1"] = {"string..."}; // strtest2 undefined on purpose - testArgs.GetOverrideArgs()["inttest1"] = {"12345"}; - testArgs.GetOverrideArgs()["inttest2"] = {"81985529216486895"}; + testArgs.m_settings.command_line_options["inttest1"] = {"12345"}; + testArgs.m_settings.command_line_options["inttest2"] = {"81985529216486895"}; // inttest3 undefined on purpose - testArgs.GetOverrideArgs()["booltest1"] = {""}; + testArgs.m_settings.command_line_options["booltest1"] = {""}; // booltest2 undefined on purpose - testArgs.GetOverrideArgs()["booltest3"] = {"0"}; - testArgs.GetOverrideArgs()["booltest4"] = {"1"}; + testArgs.m_settings.command_line_options["booltest3"] = {"0"}; + testArgs.m_settings.command_line_options["booltest4"] = {"1"}; // priorities - testArgs.GetOverrideArgs()["pritest1"] = {"a", "b"}; - testArgs.GetConfigArgs()["pritest2"] = {"a", "b"}; - testArgs.GetOverrideArgs()["pritest3"] = {"a"}; - testArgs.GetConfigArgs()["pritest3"] = {"b"}; - testArgs.GetOverrideArgs()["pritest4"] = {"a","b"}; - testArgs.GetConfigArgs()["pritest4"] = {"c","d"}; + testArgs.m_settings.command_line_options["pritest1"] = {"a", "b"}; + testArgs.m_settings.ro_config[""]["pritest2"] = {"a", "b"}; + testArgs.m_settings.command_line_options["pritest3"] = {"a"}; + testArgs.m_settings.ro_config[""]["pritest3"] = {"b"}; + testArgs.m_settings.command_line_options["pritest4"] = {"a","b"}; + testArgs.m_settings.ro_config[""]["pritest4"] = {"c","d"}; BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 3431eedff6..1fa48b325c 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -40,9 +40,10 @@ struct TestSubscriber : public CValidationInterface { m_expected_tip = block->GetHash(); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override { BOOST_CHECK_EQUAL(m_expected_tip, block->GetHash()); + BOOST_CHECK_EQUAL(m_expected_tip, pindex->GetBlockHash()); m_expected_tip = block->hashPrevBlock; } diff --git a/src/txmempool.h b/src/txmempool.h index 9ccede9d4d..4a7640b78a 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -50,8 +50,6 @@ struct LockPoints LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } }; -class CTxMemPool; - /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well diff --git a/src/util/fees.cpp b/src/util/fees.cpp index cf16d5e44f..41149888d7 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -3,8 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <util/fees.h> + #include <policy/fees.h> +#include <map> #include <string> std::string StringForFeeReason(FeeReason reason) { diff --git a/src/util/settings.cpp b/src/util/settings.cpp new file mode 100644 index 0000000000..af75fef310 --- /dev/null +++ b/src/util/settings.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <univalue.h> + +namespace util { +namespace { + +enum class Source { + FORCED, + COMMAND_LINE, + CONFIG_FILE_NETWORK_SECTION, + CONFIG_FILE_DEFAULT_SECTION +}; + +//! Merge settings from multiple sources in precedence order: +//! Forced config > command line > config file network-specific section > config file default section +//! +//! This function is provided with a callback function fn that contains +//! specific logic for how to merge the sources. +template <typename Fn> +static void MergeSettings(const Settings& settings, const std::string& section, const std::string& name, Fn&& fn) +{ + // Merge in the forced settings + if (auto* value = FindKey(settings.forced_settings, name)) { + fn(SettingsSpan(*value), Source::FORCED); + } + // Merge in the command-line options + if (auto* values = FindKey(settings.command_line_options, name)) { + fn(SettingsSpan(*values), Source::COMMAND_LINE); + } + // Merge in the network-specific section of the config file + if (!section.empty()) { + if (auto* map = FindKey(settings.ro_config, section)) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_NETWORK_SECTION); + } + } + } + // Merge in the default section of the config file + if (auto* map = FindKey(settings.ro_config, "")) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_DEFAULT_SECTION); + } + } +} +} // namespace + +SettingsValue GetSetting(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config, + bool get_chain_name) +{ + SettingsValue result; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply negated + // setting even if non-negated setting would be ignored. A negated + // value in the default section is applied to network specific options, + // even though normal non-negated values there would be ignored. + const bool never_ignore_negated_setting = span.last_negated(); + + // Weird behavior preserved for backwards compatibility: Take first + // assigned value instead of last. In general, later settings take + // precedence over early settings, but for backwards compatibility in + // the config file the precedence is reversed for all settings except + // chain name settings. + const bool reverse_precedence = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !get_chain_name; + + // Weird behavior preserved for backwards compatibility: Negated + // -regtest and -testnet arguments which you would expect to override + // values set in the configuration file are currently accepted but + // silently ignored. It would be better to apply these just like other + // negated values, or at least warn they are ignored. + const bool skip_negated_command_line = get_chain_name; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION && !never_ignore_negated_setting) return; + + // Skip negated command line settings. + if (skip_negated_command_line && span.last_negated()) return; + + // Stick with highest priority value, keeping result if already set. + if (!result.isNull()) return; + + if (!span.empty()) { + result = reverse_precedence ? span.begin()[0] : span.end()[-1]; + } else if (span.last_negated()) { + result = false; + } + }); + return result; +} + +std::vector<SettingsValue> GetSettingsList(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config) +{ + std::vector<SettingsValue> result; + bool result_complete = false; + bool prev_negated_empty = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply config + // file settings even if negated on command line. Negating a setting on + // command line will ignore earlier settings on the command line and + // ignore settings in the config file, unless the negated command line + // value is followed by non-negated value, in which case config file + // settings will be brought back from the dead (but earlier command + // line settings will still be ignored). + const bool add_zombie_config_values = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !prev_negated_empty; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION) return; + + // Add new settings to the result if isn't already complete, or if the + // values are zombies. + if (!result_complete || add_zombie_config_values) { + for (const auto& value : span) { + if (value.isArray()) { + result.insert(result.end(), value.getValues().begin(), value.getValues().end()); + } else { + result.push_back(value); + } + } + } + + // If a setting was negated, or if a setting was forced, set + // result_complete to true to ignore any later lower priority settings. + result_complete |= span.negated() > 0 || source == Source::FORCED; + + // Update the negated and empty state used for the zombie values check. + prev_negated_empty |= span.last_negated() && result.empty(); + }); + return result; +} + +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name) +{ + bool has_default_section_setting = false; + bool has_other_setting = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + if (span.empty()) return; + else if (source == Source::CONFIG_FILE_DEFAULT_SECTION) has_default_section_setting = true; + else has_other_setting = true; + }); + // If a value is set in the default section and not explicitly overwritten by the + // user on the command line or in a different section, then we want to enable + // warnings about the value being ignored. + return has_default_section_setting && !has_other_setting; +} + +SettingsSpan::SettingsSpan(const std::vector<SettingsValue>& vec) noexcept : SettingsSpan(vec.data(), vec.size()) {} +const SettingsValue* SettingsSpan::begin() const { return data + negated(); } +const SettingsValue* SettingsSpan::end() const { return data + size; } +bool SettingsSpan::empty() const { return size == 0 || last_negated(); } +bool SettingsSpan::last_negated() const { return size > 0 && data[size - 1].isFalse(); } +size_t SettingsSpan::negated() const +{ + for (size_t i = size; i > 0; --i) { + if (data[i - 1].isFalse()) return i; // Return number of negated values (position of last false value) + } + return 0; +} + +} // namespace util diff --git a/src/util/settings.h b/src/util/settings.h new file mode 100644 index 0000000000..17832e4d2c --- /dev/null +++ b/src/util/settings.h @@ -0,0 +1,87 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SETTINGS_H +#define BITCOIN_UTIL_SETTINGS_H + +#include <map> +#include <string> +#include <vector> + +class UniValue; + +namespace util { + +//! Settings value type (string/integer/boolean/null variant). +//! +//! @note UniValue is used here for convenience and because it can be easily +//! serialized in a readable format. But any other variant type that can +//! be assigned strings, int64_t, and bool values and has get_str(), +//! get_int64(), get_bool(), isNum(), isBool(), isFalse(), isTrue() and +//! isNull() methods can be substituted if there's a need to move away +//! from UniValue. (An implementation with boost::variant was posted at +//! https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812) +using SettingsValue = UniValue; + +//! Stored bitcoin settings. This struct combines settings from the command line +//! and a read-only configuration file. +struct Settings { + //! Map of setting name to forced setting value. + std::map<std::string, SettingsValue> forced_settings; + //! Map of setting name to list of command line values. + std::map<std::string, std::vector<SettingsValue>> command_line_options; + //! Map of config section name and setting name to list of config file values. + std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config; +}; + +//! Get settings value from combined sources: forced settings, command line +//! arguments and the read-only config file. +//! +//! @param ignore_default_section_config - ignore values in the default section +//! of the config file (part before any +//! [section] keywords) +//! @param get_chain_name - enable special backwards compatible behavior +//! for GetChainName +SettingsValue GetSetting(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config, bool get_chain_name); + +//! Get combined setting value similar to GetSetting(), except if setting was +//! specified multiple times, return a list of all the values specified. +std::vector<SettingsValue> GetSettingsList(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config); + +//! Return true if a setting is set in the default config file section, and not +//! overridden by a higher priority command-line or network section value. +//! +//! This is used to provide user warnings about values that might be getting +//! ignored unintentionally. +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name); + +//! Accessor for list of settings that skips negated values when iterated over. +//! The last boolean `false` value in the list and all earlier values are +//! considered negated. +struct SettingsSpan { + explicit SettingsSpan() = default; + explicit SettingsSpan(const SettingsValue& value) noexcept : SettingsSpan(&value, 1) {} + explicit SettingsSpan(const SettingsValue* data, size_t size) noexcept : data(data), size(size) {} + explicit SettingsSpan(const std::vector<SettingsValue>& vec) noexcept; + const SettingsValue* begin() const; //<! Pointer to first non-negated value. + const SettingsValue* end() const; //<! Pointer to end of values. + bool empty() const; //<! True if there are any non-negated values. + bool last_negated() const; //<! True if the last value is negated. + size_t negated() const; //<! Number of negated values. + + const SettingsValue* data = nullptr; + size_t size = 0; +}; + +//! Map lookup helper. +template <typename Map, typename Key> +auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) +{ + auto it = map.find(key); + return it == map.end() ? nullptr : &it->second; +} + +} // namespace util + +#endif // BITCOIN_UTIL_SETTINGS_H diff --git a/src/util/system.cpp b/src/util/system.cpp index 7da408eda5..2a2ae6fdf5 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -63,6 +63,7 @@ #endif #include <thread> +#include <univalue.h> // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); @@ -161,11 +162,14 @@ static bool InterpretBool(const std::string& strValue) return (atoi(strValue) != 0); } +static std::string SettingName(const std::string& arg) +{ + return arg.size() > 0 && arg[0] == '-' ? arg.substr(1) : arg; +} + /** Internal helper functions for ArgsManager */ class ArgsManagerHelper { public: - typedef std::map<std::string, std::vector<std::string>> MapArgs; - /** Determine whether to use config settings in the default section, * See also comments around ArgsManager::ArgsManager() below. */ static inline bool UseDefaultSection(const ArgsManager& am, const std::string& arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) @@ -173,91 +177,10 @@ public: return (am.m_network == CBaseChainParams::MAIN || am.m_network_only_args.count(arg) == 0); } - /** Convert regular argument into the network-specific setting */ - static inline std::string NetworkArg(const ArgsManager& am, const std::string& arg) - { - assert(arg.length() > 1 && arg[0] == '-'); - return "-" + am.m_network + "." + arg.substr(1); - } - - /** Find arguments in a map and add them to a vector */ - static inline void AddArgs(std::vector<std::string>& res, const MapArgs& map_args, const std::string& arg) - { - auto it = map_args.find(arg); - if (it != map_args.end()) { - res.insert(res.end(), it->second.begin(), it->second.end()); - } - } - - /** Return true/false if an argument is set in a map, and also - * return the first (or last) of the possibly multiple values it has - */ - static inline std::pair<bool,std::string> GetArgHelper(const MapArgs& map_args, const std::string& arg, bool getLast = false) - { - auto it = map_args.find(arg); - - if (it == map_args.end() || it->second.empty()) { - return std::make_pair(false, std::string()); - } - - if (getLast) { - return std::make_pair(true, it->second.back()); - } else { - return std::make_pair(true, it->second.front()); - } - } - - /* Get the string value of an argument, returning a pair of a boolean - * indicating the argument was found, and the value for the argument - * if it was found (or the empty string if not found). - */ - static inline std::pair<bool,std::string> GetArg(const ArgsManager &am, const std::string& arg) + static util::SettingsValue Get(const ArgsManager& am, const std::string& arg) { LOCK(am.cs_args); - std::pair<bool,std::string> found_result(false, std::string()); - - // We pass "true" to GetArgHelper in order to return the last - // argument value seen from the command line (so "bitcoind -foo=bar - // -foo=baz" gives GetArg(am,"foo")=={true,"baz"} - found_result = GetArgHelper(am.m_override_args, arg, true); - if (found_result.first) { - return found_result; - } - - // But in contrast we return the first argument seen in a config file, - // so "foo=bar \n foo=baz" in the config file gives - // GetArg(am,"foo")={true,"bar"} - if (!am.m_network.empty()) { - found_result = GetArgHelper(am.m_config_args, NetworkArg(am, arg)); - if (found_result.first) { - return found_result; - } - } - - if (UseDefaultSection(am, arg)) { - found_result = GetArgHelper(am.m_config_args, arg); - if (found_result.first) { - return found_result; - } - } - - return found_result; - } - - /* Special test for -testnet and -regtest args, because we - * don't want to be confused by craziness like "[regtest] testnet=1" - */ - static inline bool GetNetBoolArg(const ArgsManager &am, const std::string& net_arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) - { - std::pair<bool,std::string> found_result(false,std::string()); - found_result = GetArgHelper(am.m_override_args, net_arg, true); - if (!found_result.first) { - found_result = GetArgHelper(am.m_config_args, net_arg, true); - if (!found_result.first) { - return false; // not set - } - } - return InterpretBool(found_result.second); // is set, so evaluate + return GetSetting(am.m_settings, am.m_network, SettingName(arg), !UseDefaultSection(am, arg), /* get_chain_name= */ false); } }; @@ -268,13 +191,12 @@ public: * checks whether there was a double-negative (-nofoo=0 -> -foo=1). * * If there was not a double negative, it removes the "no" from the key - * and clears the args vector to indicate a negated option. + * and returns false. * - * If there was a double negative, it removes "no" from the key, sets the - * value to "1" and pushes the key and the updated value to the args vector. + * If there was a double negative, it removes "no" from the key, and + * returns true. * - * If there was no "no", it leaves key and value untouched and pushes them - * to the args vector. + * If there was no "no", it returns the string value untouched. * * Where an option was negated can be later checked using the * IsArgNegated() method. One use case for this is to have a way to disable @@ -282,34 +204,39 @@ public: * that debug log output is not sent to any file at all). */ -NODISCARD static bool InterpretOption(std::string key, std::string val, unsigned int flags, - std::map<std::string, std::vector<std::string>>& args, - std::string& error) +static util::SettingsValue InterpretOption(std::string& section, std::string& key, const std::string& value) { - assert(key[0] == '-'); - + // Split section name from key name for keys like "testnet.foo" or "regtest.bar" size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; + if (option_index != std::string::npos) { + section = key.substr(0, option_index); + key.erase(0, option_index + 1); } - if (key.substr(option_index, 2) == "no") { - key.erase(option_index, 2); - if (flags & ArgsManager::ALLOW_BOOL) { - if (InterpretBool(val)) { - args[key].clear(); - return true; - } - // Double negatives like -nofoo=0 are supported (but discouraged) - LogPrintf("Warning: parsed potentially confusing double-negative %s=%s\n", key, val); - val = "1"; - } else { - error = strprintf("Negating of %s is meaningless and therefore forbidden", key); - return false; + if (key.substr(0, 2) == "no") { + key.erase(0, 2); + // Double negatives like -nofoo=0 are supported (but discouraged) + if (!InterpretBool(value)) { + LogPrintf("Warning: parsed potentially confusing double-negative -%s=%s\n", key, value); + return true; } + return false; + } + return value; +} + +/** + * Check settings value validity according to flags. + * + * TODO: Add more meaningful error checks here in the future + * See "here's how the flags are meant to behave" in + * https://github.com/bitcoin/bitcoin/pull/16097#issuecomment-514627823 + */ +static bool CheckValid(const std::string& key, const util::SettingsValue& val, unsigned int flags, std::string& error) +{ + if (val.isBool() && !(flags & ArgsManager::ALLOW_BOOL)) { + error = strprintf("Negating of -%s is meaningless and therefore forbidden", key); + return false; } - args[key].push_back(val); return true; } @@ -331,22 +258,9 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const if (m_network == CBaseChainParams::MAIN) return std::set<std::string> {}; for (const auto& arg : m_network_only_args) { - std::pair<bool, std::string> found_result; - - // if this option is overridden it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_override_args, arg); - if (found_result.first) continue; - - // if there's a network-specific value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, ArgsManagerHelper::NetworkArg(*this, arg)); - if (found_result.first) continue; - - // if there isn't a default value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, arg); - if (!found_result.first) continue; - - // otherwise, issue a warning - unsuitables.insert(arg); + if (OnlyHasDefaultSectionSetting(m_settings, m_network, SettingName(arg))) { + unsuitables.insert(arg); + } } return unsuitables; } @@ -375,7 +289,7 @@ void ArgsManager::SelectConfigNetwork(const std::string& network) bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::string& error) { LOCK(cs_args); - m_override_args.clear(); + m_settings.command_line_options.clear(); for (int i = 1; i < argc; i++) { std::string key(argv[i]); @@ -408,49 +322,44 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin if (key.length() > 1 && key[1] == '-') key.erase(0, 1); + // Transform -foo to foo + key.erase(0, 1); + std::string section; + util::SettingsValue value = InterpretOption(section, key, val); const unsigned int flags = FlagsOfKnownArg(key); if (flags) { - if (!InterpretOption(key, val, flags, m_override_args, error)) { + if (!CheckValid(key, value, flags, error)) { return false; } + // Weird behavior preserved for backwards compatibility: command + // line options with section prefixes are allowed but ignored. It + // would be better if these options triggered the Invalid parameter + // error below. + if (section.empty()) { + m_settings.command_line_options[key].push_back(value); + } } else { - error = strprintf("Invalid parameter %s", key); + error = strprintf("Invalid parameter -%s", key); return false; } } // we do not allow -includeconf from command line, so we clear it here - auto it = m_override_args.find("-includeconf"); - if (it != m_override_args.end()) { - if (it->second.size() > 0) { - for (const auto& ic : it->second) { - error += "-includeconf cannot be used from commandline; -includeconf=" + ic + "\n"; - } - return false; + bool success = true; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + for (const auto& include : util::SettingsSpan(*includes)) { + error += "-includeconf cannot be used from commandline; -includeconf=" + include.get_str() + "\n"; + success = false; } } - return true; + return success; } unsigned int ArgsManager::FlagsOfKnownArg(const std::string& key) const { - assert(key[0] == '-'); - - size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; - } - if (key.substr(option_index, 2) == "no") { - option_index += 2; - } - - const std::string base_arg_name = '-' + key.substr(option_index); - LOCK(cs_args); for (const auto& arg_map : m_available_args) { - const auto search = arg_map.second.find(base_arg_name); + const auto search = arg_map.second.find('-' + key); if (search != arg_map.second.end()) { return search->second.m_flags; } @@ -460,69 +369,42 @@ unsigned int ArgsManager::FlagsOfKnownArg(const std::string& key) const std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { - std::vector<std::string> result = {}; - if (IsArgNegated(strArg)) return result; // special case - LOCK(cs_args); - - ArgsManagerHelper::AddArgs(result, m_override_args, strArg); - if (!m_network.empty()) { - ArgsManagerHelper::AddArgs(result, m_config_args, ArgsManagerHelper::NetworkArg(*this, strArg)); + bool ignore_default_section_config = !ArgsManagerHelper::UseDefaultSection(*this, strArg); + std::vector<std::string> result; + for (const util::SettingsValue& value : + util::GetSettingsList(m_settings, m_network, SettingName(strArg), ignore_default_section_config)) { + result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } - - if (ArgsManagerHelper::UseDefaultSection(*this, strArg)) { - ArgsManagerHelper::AddArgs(result, m_config_args, strArg); - } - return result; } bool ArgsManager::IsArgSet(const std::string& strArg) const { - if (IsArgNegated(strArg)) return true; // special case - return ArgsManagerHelper::GetArg(*this, strArg).first; + return !ArgsManagerHelper::Get(*this, strArg).isNull(); } bool ArgsManager::IsArgNegated(const std::string& strArg) const { - LOCK(cs_args); - - const auto& ov = m_override_args.find(strArg); - if (ov != m_override_args.end()) return ov->second.empty(); - - if (!m_network.empty()) { - const auto& cfs = m_config_args.find(ArgsManagerHelper::NetworkArg(*this, strArg)); - if (cfs != m_config_args.end()) return cfs->second.empty(); - } - - const auto& cf = m_config_args.find(strArg); - if (cf != m_config_args.end()) return cf->second.empty(); - - return false; + return ArgsManagerHelper::Get(*this, strArg).isFalse(); } std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) const { - if (IsArgNegated(strArg)) return "0"; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return found_res.second; - return strDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); } int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) const { - if (IsArgNegated(strArg)) return 0; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return atoi64(found_res.second); - return nDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const { - if (IsArgNegated(strArg)) return false; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return InterpretBool(found_res.second); - return fDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? fDefault : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); } bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) @@ -544,7 +426,7 @@ bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); - m_override_args[strArg] = {strValue}; + m_settings.forced_settings[SettingName(strArg)] = strValue; } void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) @@ -860,12 +742,15 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file return false; } for (const std::pair<std::string, std::string>& option : options) { - const std::string strKey = std::string("-") + option.first; - const unsigned int flags = FlagsOfKnownArg(strKey); + std::string section; + std::string key = option.first; + util::SettingsValue value = InterpretOption(section, key, option.second); + const unsigned int flags = FlagsOfKnownArg(key); if (flags) { - if (!InterpretOption(strKey, option.second, flags, m_config_args, error)) { + if (!CheckValid(key, value, flags, error)) { return false; } + m_settings.ro_config[section][key].push_back(value); } else { if (ignore_invalid_keys) { LogPrintf("Ignoring unknown configuration value %s\n", option.first); @@ -882,7 +767,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } @@ -894,58 +779,64 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } - // if there is an -includeconf in the override args, but it is empty, that means the user - // passed '-noincludeconf' on the command line, in which case we should not include anything - bool emptyIncludeConf; + // `-includeconf` cannot be included in the command line arguments except + // as `-noincludeconf` (which indicates that no conf file should be used). + bool use_conf_file{true}; { LOCK(cs_args); - emptyIncludeConf = m_override_args.count("-includeconf") == 0; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + // ParseParameters() fails if a non-negated -includeconf is passed on the command-line + assert(util::SettingsSpan(*includes).last_negated()); + use_conf_file = false; + } } - if (emptyIncludeConf) { + if (use_conf_file) { std::string chain_id = GetChainName(); - std::vector<std::string> includeconf(GetArgs("-includeconf")); - { - // We haven't set m_network yet (that happens in SelectParams()), so manually check - // for network.includeconf args. - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + std::vector<std::string> conf_file_names; - // Remove -includeconf from configuration, so we can warn about recursion - // later - { + auto add_includes = [&](const std::string& network, size_t skip = 0) { + size_t num_values = 0; LOCK(cs_args); - m_config_args.erase("-includeconf"); - m_config_args.erase(std::string("-") + chain_id + ".includeconf"); - } - - for (const std::string& to_include : includeconf) { - fsbridge::ifstream include_config(GetConfigFile(to_include)); - if (include_config.good()) { - if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { + if (auto* section = util::FindKey(m_settings.ro_config, network)) { + if (auto* values = util::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + conf_file_names.push_back((*values)[i].get_str()); + } + num_values = values->size(); + } + } + return num_values; + }; + + // We haven't set m_network yet (that happens in SelectParams()), so manually check + // for network.includeconf args. + const size_t chain_includes = add_includes(chain_id); + const size_t default_includes = add_includes({}); + + for (const std::string& conf_file_name : conf_file_names) { + fsbridge::ifstream conf_file_stream(GetConfigFile(conf_file_name)); + if (conf_file_stream.good()) { + if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; } - LogPrintf("Included configuration file %s\n", to_include); + LogPrintf("Included configuration file %s\n", conf_file_name); } else { - error = "Failed to include configuration file " + to_include; + error = "Failed to include configuration file " + conf_file_name; return false; } } // Warn about recursive -includeconf - includeconf = GetArgs("-includeconf"); - { - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - std::string chain_id_final = GetChainName(); - if (chain_id_final != chain_id) { - // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs - includeconf_net = GetArgs(std::string("-") + chain_id_final + ".includeconf"); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + conf_file_names.clear(); + add_includes(chain_id, /* skip= */ chain_includes); + add_includes({}, /* skip= */ default_includes); + std::string chain_id_final = GetChainName(); + if (chain_id_final != chain_id) { + // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs + add_includes(chain_id_final); } - for (const std::string& to_include : includeconf) { - tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", to_include); + for (const std::string& conf_file_name : conf_file_names) { + tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", conf_file_name); } } } @@ -961,9 +852,16 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) std::string ArgsManager::GetChainName() const { - LOCK(cs_args); - const bool fRegTest = ArgsManagerHelper::GetNetBoolArg(*this, "-regtest"); - const bool fTestNet = ArgsManagerHelper::GetNetBoolArg(*this, "-testnet"); + auto get_net = [&](const std::string& arg) { + LOCK(cs_args); + util::SettingsValue value = GetSetting(m_settings, /* section= */ "", SettingName(arg), + /* ignore_default_section_config= */ false, + /* get_chain_name= */ true); + return value.isNull() ? false : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); + }; + + const bool fRegTest = get_net("-regtest"); + const bool fTestNet = get_net("-testnet"); const bool is_chain_arg_set = IsArgSet("-chain"); if ((int)is_chain_arg_set + (int)fRegTest + (int)fTestNet > 1) { diff --git a/src/util/system.h b/src/util/system.h index 7452f186e6..e0b6371dc9 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -22,6 +22,7 @@ #include <sync.h> #include <tinyformat.h> #include <util/memory.h> +#include <util/settings.h> #include <util/threadnames.h> #include <util/time.h> @@ -157,8 +158,7 @@ protected: }; mutable CCriticalSection cs_args; - std::map<std::string, std::vector<std::string>> m_override_args GUARDED_BY(cs_args); - std::map<std::string, std::vector<std::string>> m_config_args GUARDED_BY(cs_args); + util::Settings m_settings GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); diff --git a/src/validation.cpp b/src/validation.cpp index 5a67493b0d..e744110371 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2458,7 +2458,7 @@ bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - GetMainSignals().BlockDisconnected(pblock); + GetMainSignals().BlockDisconnected(pblock, pindexDelete); return true; } @@ -5089,7 +5089,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; } - return pindex->nChainTx / fTxTotal; + return std::min<double>(pindex->nChainTx / fTxTotal, 1.0); } class CMainCleanup diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index a46b4003f1..663308bae9 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -29,7 +29,7 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const std::shared_ptr<const CBlock>&, const CBlockIndex* pindex)> BlockDisconnected; boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; boost::signals2::signal<void (const CBlock&, const BlockValidationState&)> BlockChecked; @@ -92,7 +92,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { conns.UpdatedBlockTip = g_signals.m_internals->UpdatedBlockTip.connect(std::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); conns.TransactionAddedToMempool = g_signals.m_internals->TransactionAddedToMempool.connect(std::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, std::placeholders::_1)); conns.BlockConnected = g_signals.m_internals->BlockConnected.connect(std::bind(&CValidationInterface::BlockConnected, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1)); + conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); @@ -156,9 +156,10 @@ void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, c }); } -void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { - m_internals->BlockDisconnected(pblock); +void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +{ + m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, this] { + m_internals->BlockDisconnected(pblock, pindex); }); } diff --git a/src/validationinterface.h b/src/validationinterface.h index dc8425869b..6a8059a6a0 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -114,7 +114,7 @@ protected: * * Called on a background thread. */ - virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) {} /** * Notifies listeners of the new active block chain on-disk. * @@ -178,7 +178,7 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); - void BlockDisconnected(const std::shared_ptr<const CBlock> &); + void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(const CBlockLocator &); void BlockChecked(const CBlock&, const BlockValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index e48eee6c2c..79825847c8 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -166,10 +166,9 @@ BerkeleyEnvironment::~BerkeleyEnvironment() bool BerkeleyEnvironment::Open(bool retry) { - if (fDbEnvInit) + if (fDbEnvInit) { return true; - - boost::this_thread::interruption_point(); + } fs::path pathIn = strPath; TryCreateDirectories(pathIn); @@ -238,14 +237,12 @@ bool BerkeleyEnvironment::Open(bool retry) return true; } -//! Construct an in-memory mock Berkeley environment for testing and as a place-holder for g_dbenvs emplace +//! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); - boost::this_thread::interruption_point(); - - LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); @@ -263,8 +260,9 @@ BerkeleyEnvironment::BerkeleyEnvironment() DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); - if (ret > 0) + if (ret > 0) { throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); + } fDbEnvInit = true; fMockDb = true; @@ -767,7 +765,7 @@ void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) return; { @@ -776,21 +774,21 @@ void BerkeleyEnvironment::Flush(bool fShutdown) while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else mi++; } - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { @@ -830,7 +828,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::DB, "Flushing %s\n", strFile); + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained @@ -838,7 +836,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 0a4bb3f396..8f0b495ac4 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -16,7 +16,7 @@ //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. -static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.GetHash())) { errors.push_back("Transaction has descendants in the wallet"); @@ -30,7 +30,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } } - if (wtx.GetDepthInMainChain(locked_chain) != 0) { + if (wtx.GetDepthInMainChain() != 0) { errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); return feebumper::Result::WALLET_ERROR; } @@ -146,7 +146,7 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) if (wtx == nullptr) return false; std::vector<std::string> errors_dummy; - feebumper::Result res = PreconditionChecks(*locked_chain, wallet, *wtx, errors_dummy); + feebumper::Result res = PreconditionChecks(wallet, *wtx, errors_dummy); return res == feebumper::Result::OK; } @@ -165,7 +165,7 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, *wallet, wtx, errors); + Result result = PreconditionChecks(*wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -291,7 +291,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, wallet, wtx, errors); + Result result = PreconditionChecks(wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -382,7 +382,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime - Result result = PreconditionChecks(*locked_chain, wallet, oldWtx, errors); + Result result = PreconditionChecks(wallet, oldWtx, errors); if (result != Result::OK) { return result; } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6995974f08..d95481500d 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -316,7 +316,7 @@ UniValue importaddress(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -354,28 +354,26 @@ UniValue importprunedfunds(const JSONRPCRequest& request) //Search partial merkle tree in proof for our transaction and index in valid block std::vector<uint256> vMatch; std::vector<unsigned int> vIndex; - unsigned int txnIndex = 0; - if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) == merkleBlock.header.hashMerkleRoot) { - - auto locked_chain = pwallet->chain().lock(); - if (locked_chain->getBlockHeight(merkleBlock.header.GetHash()) == nullopt) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - } - - std::vector<uint256>::const_iterator it; - if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx))==vMatch.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); - } + if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + } - txnIndex = vIndex[it - vMatch.begin()]; + auto locked_chain = pwallet->chain().lock(); + Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); + if (height == nullopt) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } - else { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + + std::vector<uint256>::const_iterator it; + if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx)) == vMatch.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); } - wtx.SetConf(CWalletTx::Status::CONFIRMED, merkleBlock.header.GetHash(), txnIndex); + unsigned int txnIndex = vIndex[it - vMatch.begin()]; + + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + wtx.m_confirm = confirm; - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (pwallet->IsMine(*wtx.tx)) { @@ -507,7 +505,7 @@ UniValue importpubkey(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -1406,7 +1404,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } if (pwallet->IsAbortingRescan()) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6098a08292..00742aed7a 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -135,13 +135,14 @@ LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet) static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) { - int confirms = wtx.GetDepthInMainChain(locked_chain); + int confirms = wtx.GetDepthInMainChain(); entry.pushKV("confirmations", confirms); if (wtx.IsCoinBase()) entry.pushKV("generated", true); if (confirms > 0) { entry.pushKV("blockhash", wtx.m_confirm.hashBlock.GetHex()); + entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); @@ -640,7 +641,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } @@ -706,7 +707,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) { CTxDestination address; if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } } @@ -1063,7 +1064,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co continue; } - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < nMinDepth) continue; @@ -1320,8 +1321,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con } // Received - if (listReceived.size() > 0 && wtx.GetDepthInMainChain(locked_chain) >= nMinDepth) - { + if (listReceived.size() > 0 && wtx.GetDepthInMainChain() >= nMinDepth) { for (const COutputEntry& r : listReceived) { std::string label; @@ -1338,9 +1338,9 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) { - if (wtx.GetDepthInMainChain(locked_chain) < 1) + if (wtx.GetDepthInMainChain() < 1) entry.pushKV("category", "orphan"); - else if (wtx.IsImmatureCoinBase(locked_chain)) + else if (wtx.IsImmatureCoinBase()) entry.pushKV("category", "immature"); else entry.pushKV("category", "generate"); @@ -1368,6 +1368,7 @@ static const std::string TransactionDescriptionString() " \"generated\": xxx, (bool) Only present if transaction only input is a coinbase one.\n" " \"trusted\": xxx, (bool) Only present if we consider transaction to be trusted and so safe to spend from.\n" " \"blockhash\": \"hashvalue\", (string) The block hash containing the transaction.\n" + " \"blockheight\": n, (numeric) The block height containing the transaction.\n" " \"blockindex\": n, (numeric) The index of the transaction in the block that includes it.\n" " \"blocktime\": xxx, (numeric) The block time in seconds since epoch (1 Jan 1970 GMT).\n" " \"txid\": \"transactionid\", (string) The transaction id.\n" @@ -1604,7 +1605,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; - if (depth == -1 || abs(tx.GetDepthInMainChain(*locked_chain)) < depth) { + if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1721,7 +1722,7 @@ static UniValue gettransaction(const JSONRPCRequest& request) } const CWalletTx& wtx = it->second; - CAmount nCredit = wtx.GetCredit(*locked_chain, filter); + CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); CAmount nNet = nCredit - nDebit; CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); @@ -1785,7 +1786,7 @@ static UniValue abandontransaction(const JSONRPCRequest& request) if (!pwallet->mapWallet.count(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); } - if (!pwallet->AbandonTransaction(*locked_chain, hash)) { + if (!pwallet->AbandonTransaction(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not eligible for abandonment"); } @@ -2216,7 +2217,7 @@ static UniValue lockunspent(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout index out of bounds"); } - if (pwallet->IsSpent(*locked_chain, outpt.hash, outpt.n)) { + if (pwallet->IsSpent(outpt.hash, outpt.n)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected unspent output"); } diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 0b4b020eb2..2f21b2439b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -50,6 +50,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions accommodates a null start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -65,6 +69,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // and new block files. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -84,6 +92,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // file. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -102,6 +114,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -258,18 +274,20 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) LockAssertion lock(::cs_main); LOCK(wallet.cs_wallet); AssertLockHeld(spk_man->cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); - wtx.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 0); + wtx.m_confirm = confirm; // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 0); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); // Invalidate the cached vanue, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 50*COIN); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) @@ -300,7 +318,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 wallet.AddToWallet(wtx); } if (block) { - wtx.SetConf(CWalletTx::Status::CONFIRMED, block->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block->nHeight, block->GetBlockHash(), 0); + wtx.m_confirm = confirm; } wallet.AddToWallet(wtx); return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; @@ -338,9 +357,10 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests) { CTxDestination dest = PKHash(); LOCK(m_wallet.cs_wallet); - m_wallet.AddDestData(dest, "misc", "val_misc"); - m_wallet.AddDestData(dest, "rr0", "val_rr0"); - m_wallet.AddDestData(dest, "rr1", "val_rr1"); + WalletBatch batch{m_wallet.GetDatabase()}; + m_wallet.AddDestData(batch, dest, "misc", "val_misc"); + m_wallet.AddDestData(batch, dest, "rr0", "val_rr0"); + m_wallet.AddDestData(batch, dest, "rr1", "val_rr1"); auto values = m_wallet.GetDestValues("rr"); BOOST_CHECK_EQUAL(values.size(), 2U); @@ -435,6 +455,10 @@ public: { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + { + LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); @@ -473,9 +497,11 @@ public: LOCK(cs_main); LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, ::ChainActive().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - it->second.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 1); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 1); + it->second.m_confirm = confirm; return it->second; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index cdcb65e3c0..ff182c847f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -452,7 +452,7 @@ void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> ran * Outpoint is spent if any non-conflicted transaction * spends it: */ -bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const +bool CWallet::IsSpent(const uint256& hash, unsigned int n) const { const COutPoint outpoint(hash, n); std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; @@ -463,7 +463,7 @@ bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash const uint256& wtxid = it->second; std::map<uint256, CWalletTx>::const_iterator mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = mit->second.GetDepthInMainChain(locked_chain); + int depth = mit->second.GetDepthInMainChain(); if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) return true; // Spent } @@ -701,19 +701,19 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) return success; } -void CWallet::SetUsedDestinationState(const uint256& hash, unsigned int n, bool used) +void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) { + AssertLockHeld(cs_wallet); const CWalletTx* srctx = GetWalletTx(hash); if (!srctx) return; CTxDestination dst; if (ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst)) { if (IsMine(dst)) { - LOCK(cs_wallet); if (used && !GetDestData(dst, "used", nullptr)) { - AddDestData(dst, "used", "p"); // p for "present", opposite of absent (null) + AddDestData(batch, dst, "used", "p"); // p for "present", opposite of absent (null) } else if (!used && GetDestData(dst, "used", nullptr)) { - EraseDestData(dst, "used"); + EraseDestData(batch, dst, "used"); } } } @@ -744,7 +744,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) // Mark used destinations for (const CTxIn& txin : wtxIn.tx->vin) { const COutPoint& op = txin.prevout; - SetUsedDestinationState(op.hash, op.n, true); + SetUsedDestinationState(batch, op.hash, op.n, true); } } @@ -768,10 +768,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.m_confirm.status = wtxIn.m_confirm.status; wtx.m_confirm.nIndex = wtxIn.m_confirm.nIndex; wtx.m_confirm.hashBlock = wtxIn.m_confirm.hashBlock; + wtx.m_confirm.block_height = wtxIn.m_confirm.block_height; fUpdated = true; } else { assert(wtx.m_confirm.nIndex == wtxIn.m_confirm.nIndex); assert(wtx.m_confirm.hashBlock == wtxIn.m_confirm.hashBlock); + assert(wtx.m_confirm.block_height == wtxIn.m_confirm.block_height); } if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) { @@ -822,12 +824,22 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) { // If wallet doesn't have a chain (e.g wallet-tool), lock can't be taken. auto locked_chain = LockChain(); - // If tx hasn't been reorged out of chain while wallet being shutdown - // change tx status to UNCONFIRMED and reset hashBlock/nIndex. - if (!wtxIn.m_confirm.hashBlock.IsNull()) { - if (locked_chain && !locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock)) { + if (locked_chain) { + Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); + if (block_height) { + // Update cached block height variable since it not stored in the + // serialized transaction. + wtxIn.m_confirm.block_height = *block_height; + } else if (wtxIn.isConflicted() || wtxIn.isConfirmed()) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. wtxIn.setUnconfirmed(); wtxIn.m_confirm.hashBlock = uint256(); + wtxIn.m_confirm.block_height = 0; wtxIn.m_confirm.nIndex = 0; } } @@ -844,25 +856,25 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; if (prevtx.isConflicted()) { - MarkConflicted(prevtx.m_confirm.hashBlock, wtx.GetHash()); + MarkConflicted(prevtx.m_confirm.hashBlock, prevtx.m_confirm.block_height, wtx.GetHash()); } } } } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) { const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (!block_hash.IsNull()) { + if (!confirm.hashBlock.IsNull()) { for (const CTxIn& txin : tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { if (range.first->second != tx.GetHash()) { - WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), block_hash.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); - MarkConflicted(block_hash, range.first->second); + WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), confirm.hashBlock.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); + MarkConflicted(confirm.hashBlock, confirm.block_height, range.first->second); } range.first++; } @@ -890,7 +902,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - wtx.SetConf(status, block_hash, posInBlock); + wtx.m_confirm = confirm; return AddToWallet(wtx, false); } @@ -903,7 +915,7 @@ bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const auto locked_chain = chain().lock(); LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); - return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain(*locked_chain) == 0 && !wtx->InMempool(); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); } void CWallet::MarkInputsDirty(const CTransactionRef& tx) @@ -916,9 +928,9 @@ void CWallet::MarkInputsDirty(const CTransactionRef& tx) } } -bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx) +bool CWallet::AbandonTransaction(const uint256& hashTx) { - auto locked_chain_recursive = chain().lock(); // Temporary. Removed in upcoming lock cleanup + auto locked_chain = chain().lock(); // Temporary. Removed in upcoming lock cleanup LOCK(cs_wallet); WalletBatch batch(*database, "r+"); @@ -930,7 +942,7 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); CWalletTx& origtx = it->second; - if (origtx.GetDepthInMainChain(locked_chain) != 0 || origtx.InMempool()) { + if (origtx.GetDepthInMainChain() != 0 || origtx.InMempool()) { return false; } @@ -943,14 +955,13 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); // If the orig tx was not in block, none of its spends can be assert(currentconfirm <= 0); // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} if (currentconfirm == 0 && !wtx.isAbandoned()) { // If the orig tx was not in block/mempool, none of its spends can be in mempool assert(!wtx.InMempool()); - wtx.m_confirm.nIndex = 0; wtx.setAbandoned(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -972,12 +983,12 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui return true; } -void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) +void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - int conflictconfirms = -locked_chain->getBlockDepth(hashBlock); + int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; // If number of conflict confirms cannot be determined, this means // that the block is still unknown or not yet part of the main chain, // for example when loading the wallet during a reindex. Do nothing in that @@ -1000,12 +1011,13 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(*locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); if (conflictconfirms < currentconfirm) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. wtx.m_confirm.nIndex = 0; wtx.m_confirm.hashBlock = hashBlock; + wtx.m_confirm.block_height = conflicting_height; wtx.setConflicted(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -1024,9 +1036,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool update_tx) +void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx) { - if (!AddToWalletIfInvolvingMe(ptx, status, block_hash, posInBlock, update_tx)) + if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1038,7 +1050,8 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status stat void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); auto it = mapWallet.find(ptx->GetHash()); if (it != mapWallet.end()) { @@ -1054,23 +1067,26 @@ void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { } } -void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) { +void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) +{ const uint256& block_hash = block.GetHash(); auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (size_t i = 0; i < block.vtx.size(); i++) { - SyncTransaction(block.vtx[i], CWalletTx::Status::CONFIRMED, block_hash, i); - TransactionRemovedFromMempool(block.vtx[i]); + m_last_block_processed_height = height; + m_last_block_processed = block_hash; + for (size_t index = 0; index < block.vtx.size(); index++) { + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, block_hash, index); + SyncTransaction(block.vtx[index], confirm); + TransactionRemovedFromMempool(block.vtx[index]); } for (const CTransactionRef& ptx : vtxConflicted) { TransactionRemovedFromMempool(ptx); } - - m_last_block_processed = block_hash; } -void CWallet::BlockDisconnected(const CBlock& block) { +void CWallet::BlockDisconnected(const CBlock& block, int height) +{ auto locked_chain = chain().lock(); LOCK(cs_wallet); @@ -1078,8 +1094,11 @@ void CWallet::BlockDisconnected(const CBlock& block) { // be unconfirmed, whether or not the transaction is added back to the mempool. // User may have to call abandontransaction again. It may be addressed in the // future with a stickier abandoned state or even removing abandontransaction call. + m_last_block_processed_height = height - 1; + m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); } } @@ -1096,7 +1115,7 @@ void CWallet::BlockUntilSyncedToCurrentChain() { // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); - chain().waitForNotificationsIfNewBlocksConnected(last_block_hash); + chain().waitForNotificationsIfTipChanged(last_block_hash); } @@ -1625,7 +1644,8 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], CWalletTx::Status::CONFIRMED, block_hash, posInBlock, fUpdate); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -1673,7 +1693,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc return result; } -void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) +void CWallet::ReacceptWalletTransactions() { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) @@ -1686,7 +1706,7 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1697,11 +1717,11 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); std::string unused_err_string; - wtx.SubmitMemoryPoolAndRelay(unused_err_string, false, locked_chain); + wtx.SubmitMemoryPoolAndRelay(unused_err_string, false); } } -bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain) +bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay) { // Can't relay if wallet is not broadcasting if (!pwallet->GetBroadcastTransactions()) return false; @@ -1711,7 +1731,7 @@ bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, in // cause log spam. if (IsCoinBase()) return false; // Don't try to submit conflicted or confirmed transactions. - if (GetDepthInMainChain(locked_chain) != 0) return false; + if (GetDepthInMainChain() != 0) return false; // Submit transaction to mempool for relay pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString()); @@ -1765,10 +1785,10 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const return debit; } -CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const +CAmount CWalletTx::GetCredit(const isminefilter& filter) const { // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; CAmount credit = 0; @@ -1782,16 +1802,16 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine return credit; } -CAmount CWalletTx::GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache) const +CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; } -CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache, const isminefilter& filter) const +CAmount CWalletTx::GetAvailableCredit(bool fUseCache, const isminefilter& filter) const { if (pwallet == nullptr) return 0; @@ -1800,7 +1820,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo bool allow_cache = (filter & ISMINE_ALL) && (filter & ISMINE_ALL) != ISMINE_ALL; // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { @@ -1812,7 +1832,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo uint256 hashTx = GetHash(); for (unsigned int i = 0; i < tx->vout.size(); i++) { - if (!pwallet->IsSpent(locked_chain, hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { + if (!pwallet->IsSpent(hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, filter); if (!MoneyRange(nCredit)) @@ -1827,9 +1847,9 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo return nCredit; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache) const +CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } @@ -1860,7 +1880,7 @@ bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint25 { // Quick answer in most cases if (!locked_chain.checkFinalTx(*tx)) return false; - int nDepth = GetDepthInMainChain(locked_chain); + int nDepth = GetDepthInMainChain(); if (nDepth >= 1) return true; if (nDepth < 0) return false; // using wtx's cached debit @@ -1936,7 +1956,7 @@ void CWallet::ResendWalletTransactions() // any confirmed or conflicting txs. if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; std::string unused_err_string; - if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true, *locked_chain)) ++submitted_tx_count; + if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; } } // locked_chain and cs_wallet @@ -1973,9 +1993,9 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons { const CWalletTx& wtx = entry.second; const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; - const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; - const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const int tx_depth{wtx.GetDepthInMainChain()}; + const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; @@ -1984,8 +2004,8 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += wtx.GetImmatureCredit(*locked_chain); - ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(*locked_chain); + ret.m_mine_immature += wtx.GetImmatureCredit(); + ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(); } } return ret; @@ -2029,10 +2049,10 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < 0) continue; @@ -2092,7 +2112,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsLockedCoin(entry.first, i)) continue; - if (IsSpent(locked_chain, wtxid, i)) + if (IsSpent(wtxid, i)) continue; isminetype mine = IsMine(wtx.tx->vout[i]); @@ -2151,7 +2171,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Ch for (const COutPoint& output : lockedCoins) { auto it = mapWallet.find(output.hash); if (it != mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0 && output.n < it->second.tx->vout.size() && IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { CTxDestination address; @@ -2891,7 +2911,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve } std::string err_string; - if (!wtx.SubmitMemoryPoolAndRelay(err_string, true, *locked_chain)) { + if (!wtx.SubmitMemoryPoolAndRelay(err_string, true)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } @@ -3111,10 +3131,10 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: if (!wtx.IsTrusted(locked_chain, trusted_parents)) continue; - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; @@ -3126,7 +3146,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; + CAmount n = IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3440,20 +3460,20 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const return nTimeSmart; } -bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value) +bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key, const std::string &value) { if (boost::get<CNoDestination>(&dest)) return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - return WalletBatch(*database).WriteDestData(EncodeDestination(dest), key, value); + return batch.WriteDestData(EncodeDestination(dest), key, value); } -bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) +bool CWallet::EraseDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - return WalletBatch(*database).EraseDestData(EncodeDestination(dest), key); + return batch.EraseDestData(EncodeDestination(dest), key); } void CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3631,9 +3651,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } if (auto spk_man = walletInstance->m_spk_man.get()) { - std::string error; if (!spk_man->Upgrade(prev_version, error)) { - chain.initError(error); return nullptr; } } @@ -3785,8 +3803,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const Optional<int> tip_height = locked_chain->getHeight(); if (tip_height) { walletInstance->m_last_block_processed = locked_chain->getBlockHash(*tip_height); + walletInstance->m_last_block_processed_height = *tip_height; } else { walletInstance->m_last_block_processed.SetNull(); + walletInstance->m_last_block_processed_height = -1; } if (tip_height && *tip_height != rescan_height) @@ -3888,7 +3908,7 @@ void CWallet::postInitProcess() // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(*locked_chain); + ReacceptWalletTransactions(); // Update wallet transactions with current mempool transactions. chain().requestMempoolTransactions(*this); @@ -3914,38 +3934,28 @@ CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) m_pre_split = false; } -void CWalletTx::SetConf(Status status, const uint256& block_hash, int posInBlock) -{ - // Update tx status - m_confirm.status = status; - - // Update the tx's hashBlock - m_confirm.hashBlock = block_hash; - - // set the position of the transaction in the block - m_confirm.nIndex = posInBlock; -} - -int CWalletTx::GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetDepthInMainChain() const { + assert(pwallet != nullptr); + AssertLockHeld(pwallet->cs_wallet); if (isUnconfirmed() || isAbandoned()) return 0; - return locked_chain.getBlockDepth(m_confirm.hashBlock) * (isConflicted() ? -1 : 1); + return (pwallet->GetLastBlockHeight() - m_confirm.block_height + 1) * (isConflicted() ? -1 : 1); } -int CWalletTx::GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetBlocksToMaturity() const { if (!IsCoinBase()) return 0; - int chain_depth = GetDepthInMainChain(locked_chain); + int chain_depth = GetDepthInMainChain(); assert(chain_depth >= 0); // coinbase tx should not be conflicted return std::max(0, (COINBASE_MATURITY+1) - chain_depth); } -bool CWalletTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const +bool CWalletTx::IsImmatureCoinBase() const { // note GetBlocksToMaturity is 0 for non-coinbase tx - return GetBlocksToMaturity(locked_chain) > 0; + return GetBlocksToMaturity() > 0; } std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6c9b3f40ab..a6b9c0131a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -356,14 +356,17 @@ public: ABANDONED }; - /* Confirmation includes tx status and a pair of {block hash/tx index in block} at which tx has been confirmed. - * This pair is both 0 if tx hasn't confirmed yet. Meaning of these fields changes with CONFLICTED state - * where they instead point to block hash and index of the deepest conflicting tx. + /* Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} + * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. + * Meaning of these fields changes with CONFLICTED state where they instead point to block hash + * and block height of the deepest conflicting tx. */ struct Confirmation { - Status status = UNCONFIRMED; - uint256 hashBlock = uint256(); - int nIndex = 0; + Status status; + int block_height; + uint256 hashBlock; + int nIndex; + Confirmation(Status s = UNCONFIRMED, int b = 0, uint256 h = uint256(), int i = 0) : status(s), block_height(b), hashBlock(h), nIndex(i) {} }; Confirmation m_confirm; @@ -406,7 +409,6 @@ public: * compatibility (pre-commit 9ac63d6). */ if (serializedIndex == -1 && m_confirm.hashBlock == ABANDON_HASH) { - m_confirm.hashBlock = uint256(); setAbandoned(); } else if (serializedIndex == -1) { setConflicted(); @@ -447,14 +449,14 @@ public: //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; - CAmount GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const; - CAmount GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true) const; + CAmount GetCredit(const isminefilter& filter) const; + CAmount GetImmatureCredit(bool fUseCache = true) const; // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid // having to resolve the issue of member access into incomplete type CWallet. - CAmount GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true, const isminefilter& filter=ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; - CAmount GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache=true) const; + CAmount GetAvailableCredit(bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; + CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; CAmount GetChange() const; // Get the marginal bytes if spending the specified output from this transaction @@ -481,7 +483,7 @@ public: int64_t GetTxTime() const; // Pass this transaction to node for mempool insertion and relay to peers if flag set to true - bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain); + bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -491,38 +493,44 @@ public: // in place. std::set<uint256> GetConflicts() const NO_THREAD_SAFETY_ANALYSIS; - void SetConf(Status status, const uint256& block_hash, int posInBlock); - /** * Return depth of transaction in blockchain: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain */ - int GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const; - bool IsInMainChain(interfaces::Chain::Lock& locked_chain) const { return GetDepthInMainChain(locked_chain) > 0; } + // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation + // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to + // resolve the issue of member access into incomplete type CWallet. Note + // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" + // in place. + int GetDepthInMainChain() const NO_THREAD_SAFETY_ANALYSIS; + bool IsInMainChain() const { return GetDepthInMainChain() > 0; } /** * @return number of blocks to maturity for this transaction: * 0 : is not a coinbase transaction, or is a mature coinbase transaction * >0 : is a coinbase transaction which matures in this many blocks */ - int GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const; + int GetBlocksToMaturity() const; bool isAbandoned() const { return m_confirm.status == CWalletTx::ABANDONED; } void setAbandoned() { m_confirm.status = CWalletTx::ABANDONED; m_confirm.hashBlock = uint256(); + m_confirm.block_height = 0; m_confirm.nIndex = 0; } bool isConflicted() const { return m_confirm.status == CWalletTx::CONFLICTED; } void setConflicted() { m_confirm.status = CWalletTx::CONFLICTED; } bool isUnconfirmed() const { return m_confirm.status == CWalletTx::UNCONFIRMED; } void setUnconfirmed() { m_confirm.status = CWalletTx::UNCONFIRMED; } + bool isConfirmed() const { return m_confirm.status == CWalletTx::CONFIRMED; } void setConfirmed() { m_confirm.status = CWalletTx::CONFIRMED; } const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } - bool IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const; + bool IsImmatureCoinBase() const; }; class COutput @@ -642,10 +650,10 @@ private: * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ - void MarkConflicted(const uint256& hashBlock, const uint256& hashTx); + void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -654,7 +662,7 @@ private: /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock = 0, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::atomic<uint64_t> m_wallet_flags{0}; @@ -679,12 +687,18 @@ private: * The following is used to keep track of how far behind the wallet is * from the chain sync, and to allow clients to block on us being caught up. * - * Note that this is *not* how far we've processed, we may need some rescan - * to have seen all transactions in the chain, but is only used to track - * live BlockConnected callbacks. + * Processed hash is a pointer on node's tip and doesn't imply that the wallet + * has scanned sequentially all blocks up to this one. */ uint256 m_last_block_processed GUARDED_BY(cs_wallet); + /* Height of last block processed is used by wallet to know depth of transactions + * without relying on Chain interface beyond asynchronous updates. For safety, we + * initialize it to -1. Height is a pointer on node's tip and doesn't imply + * that the wallet has scanned sequentially all blocks up to this one. + */ + int m_last_block_processed_height GUARDED_BY(cs_wallet) = -1; + public: /* * Main wallet lock. @@ -794,12 +808,12 @@ public: bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; - bool IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); // Whether this or any UTXO with the same CTxDestination has been spent. bool IsUsedDestination(const CTxDestination& dst) const; bool IsUsedDestination(const uint256& hash, unsigned int n) const; - void SetUsedDestinationState(const uint256& hash, unsigned int n, bool used); + void SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; @@ -824,9 +838,9 @@ public: bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } //! Adds a destination data tuple to the store, and saves it to disk - bool AddDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk - bool EraseDestData(const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool EraseDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a destination data tuple to the store, without saving it to disk void LoadDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Look up a destination data tuple in the store, return true if found false otherwise @@ -855,8 +869,8 @@ public: bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); void LoadToWallet(CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void TransactionAddedToMempool(const CTransactionRef& tx) override; - void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const CBlock& block) override; + void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) override; + void BlockDisconnected(const CBlock& block, int height) override; void UpdatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); @@ -877,7 +891,7 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); struct Balance { CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more @@ -1056,7 +1070,7 @@ public: bool TransactionCanBeAbandoned(const uint256& hashTx) const; /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ - bool AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx); + bool AbandonTransaction(const uint256& hashTx); /** Mark a transaction as replaced by another transaction (e.g., BIP 125). */ bool MarkReplaced(const uint256& originalHash, const uint256& newHash); @@ -1129,6 +1143,21 @@ public: LegacyScriptPubKeyMan::WatchKeyMap& mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys; WalletBatch*& encrypted_batch GUARDED_BY(cs_wallet) = m_spk_man->encrypted_batch; using CryptedKeyMap = LegacyScriptPubKeyMan::CryptedKeyMap; + + /** Get last block processed height */ + int GetLastBlockHeight() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed_height; + }; + /** Set last block processed height, currently only use in unit test */ + void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + m_last_block_processed_height = block_height; + m_last_block_processed = block_hash; + }; }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 2ba7cdac36..7d04b04764 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -629,7 +629,7 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u } else if ((*it) == hash) { if(!EraseTx(hash)) { - LogPrint(BCLog::DB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); + LogPrint(BCLog::WALLETDB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); delerror = true; } vTxHashOut.push_back(hash); diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index ebbaf8683d..b3b97b6a2a 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -185,7 +185,7 @@ void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBloc } } -void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) { for (const CTransactionRef& ptx : pblock->vtx) { // Do a normal notify for each transaction removed in block disconnection diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 6be0554a65..8bf9b0ba47 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -27,7 +27,7 @@ protected: // CValidationInterface void TransactionAddedToMempool(const CTransactionRef& tx) override; void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: diff --git a/test/config.ini.in b/test/config.ini.in index 060c553da2..9687206ee1 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -16,6 +16,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py # Which components are enabled. These are commented out by `configure` if they were disabled when running config. @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 0a378c5ef5..ec1c88ed53 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -12,6 +12,9 @@ class TestBitcoinCli(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + def run_test(self): """Main test logic""" diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index c466b120f2..0eef1d3ddf 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -267,6 +267,12 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) + tx.vin[0].scriptSig = CScript([b'a' * 1648]) # Some too large scriptSig (>1650 bytes) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptsig-size'}], + rawtxs=[tx.serialize().hex()], + ) + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) num_scripts = 100000 // len(output_p2sh_burn.serialize()) # Use enough outputs to make the tx too large for our policy tx.vout = [output_p2sh_burn] * num_scripts diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c7d241503a..7014105d88 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -14,13 +14,19 @@ from test_framework.util import ( satoshi_round, ) +# default limits MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 +# custom limits for node1 +MAX_ANCESTORS_CUSTOM = 5 class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.extra_args = [["-maxorphantx=1000"], ["-maxorphantx=1000", "-limitancestorcount=5"]] + self.extra_args = [ + ["-maxorphantx=1000"], + ["-maxorphantx=1000", "-limitancestorcount={}".format(MAX_ANCESTORS_CUSTOM)], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -188,7 +194,14 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000) assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) - # TODO: check that node1's mempool is as expected + # Check that node1's mempool is as expected (-> custom ancestor limit) + mempool0 = self.nodes[0].getrawmempool(False) + mempool1 = self.nodes[1].getrawmempool(False) + assert_equal(len(mempool1), MAX_ANCESTORS_CUSTOM) + assert set(mempool1).issubset(set(mempool0)) + for tx in chain[:MAX_ANCESTORS_CUSTOM]: + assert tx in mempool1 + # TODO: more detailed check of node1's mempool (fees etc.) # TODO: test ancestor size limits diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index f0ceb8e6a3..20864881c1 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" import asyncio -import os import struct import sys @@ -66,27 +65,21 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) assert len(msg_at_size.serialize()) == msg_limit - increase_allowed = 0.5 - if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]: - increase_allowed = 3.5 - with node.assert_memory_usage_stable(increase_allowed=increase_allowed): - self.log.info( - "Sending a bunch of large, junk messages to test " - "memory exhaustion. May take a bit...") - - # Run a bunch of times to test for memory exhaustion. - for _ in range(80): - node.p2p.send_message(msg_at_size) - - # Check that, even though the node is being hammered by nonsense from one - # connection, it can still service other peers in a timely way. - for _ in range(20): - conn2.sync_with_ping(timeout=2) - - # Peer 1, despite serving up a bunch of nonsense, should still be connected. - self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=320) - assert node.p2p.is_connected + self.log.info("Sending a bunch of large, junk messages to test memory exhaustion. May take a bit...") + + # Run a bunch of times to test for memory exhaustion. + for _ in range(80): + node.p2p.send_message(msg_at_size) + + # Check that, even though the node is being hammered by nonsense from one + # connection, it can still service other peers in a timely way. + for _ in range(20): + conn2.sync_with_ping(timeout=2) + + # Peer 1, despite serving up a bunch of nonsense, should still be connected. + self.log.info("Waiting for node to drop junk messages.") + node.p2p.sync_with_ping(timeout=320) + assert node.p2p.is_connected # # 1. diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 0c7edbf434..fdb70097ee 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1116,7 +1116,7 @@ class SegWitTest(BitcoinTestFramework): MAX_PROGRAM_LENGTH = 10000 # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. - long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) + long_witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 63 + [OP_TRUE]) assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1140,7 +1140,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Try again with one less byte in the witness program - witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) + witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index c56c0d06ff..6b6bbfd1f9 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -596,6 +596,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") + def skip_if_no_wallet_tool(self): + """Skip the running test if bitcoin-wallet has not been compiled.""" + if not self.is_wallet_tool_compiled(): + raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -609,6 +614,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether the wallet module was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET") + def is_wallet_tool_compiled(self): + """Checks whether bitcoin-wallet was compiled.""" + return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d8bfdfd040..1c9628264f 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -135,25 +135,6 @@ class TestNode(): assert len(self.PRIV_KEYS) == MAX_NODES return self.PRIV_KEYS[self.index] - def get_mem_rss_kilobytes(self): - """Get the memory usage (RSS) per `ps`. - - Returns None if `ps` is unavailable. - """ - assert self.running - - try: - return int(subprocess.check_output( - ["ps", "h", "-o", "rss", "{}".format(self.process.pid)], - stderr=subprocess.DEVNULL).split()[-1]) - - # Avoid failing on platforms where ps isn't installed. - # - # We could later use something like `psutils` to work across platforms. - except (FileNotFoundError, subprocess.SubprocessError): - self.log.exception("Unable to get memory usage") - return None - def _node_msg(self, msg: str) -> str: """Return a modified msg that identifies this node by its index as a debugging aid.""" return "[node %d] %s" % (self.index, msg) @@ -333,33 +314,6 @@ class TestNode(): self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager - def assert_memory_usage_stable(self, *, increase_allowed=0.03): - """Context manager that allows the user to assert that a node's memory usage (RSS) - hasn't increased beyond some threshold percentage. - - Args: - increase_allowed (float): the fractional increase in memory allowed until failure; - e.g. `0.12` for up to 12% increase allowed. - """ - before_memory_usage = self.get_mem_rss_kilobytes() - - yield - - after_memory_usage = self.get_mem_rss_kilobytes() - - if not (before_memory_usage and after_memory_usage): - self.log.warning("Unable to detect memory usage (RSS) - skipping memory check.") - return - - perc_increase_memory_usage = (after_memory_usage / before_memory_usage) - 1 - - if perc_increase_memory_usage > increase_allowed: - self._raise_assertion_error( - "Memory usage increased over threshold of {:.3f}% from {} to {} ({:.3f}%)".format( - increase_allowed * 100, before_memory_usage, after_memory_usage, - perc_increase_memory_usage * 100)) - - @contextlib.contextmanager def profile_with_perf(self, profile_name): """ Context manager that allows easy profiling of node activity using `perf`. diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9b4a5d2030..4156e22720 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -364,9 +364,10 @@ def main(): def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, runs_ci, use_term_control): args = args or [] - # Warn if bitcoind is already running (unix only) + # Warn if bitcoind is already running + # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) is not None: + if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) except (OSError, subprocess.SubprocessError): pass diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 355cd7af75..32ef257456 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -23,6 +23,7 @@ class ToolWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_wallet_tool() def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 95d51adebb..9d6aa36c35 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -211,15 +211,15 @@ def test_small_output_with_feerate_succeeds(rbf_node, dest_address): # Make sure additional inputs exist rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) rbfid = spend_one_input(rbf_node, dest_address) - original_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - assert_equal(len(original_input_list), 1) - original_txin = original_input_list[0] + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_equal(len(input_list), 1) + original_txin = input_list[0] # Keep bumping until we out-spend change output tx_fee = 0 while tx_fee < Decimal("0.0005"): - new_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - new_item = list(new_input_list)[0] - assert_equal(len(original_input_list), 1) + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + new_item = list(input_list)[0] + assert_equal(len(input_list), 1) assert_equal(original_txin["txid"], new_item["txid"]) assert_equal(original_txin["vout"], new_item["vout"]) rbfid_new_details = rbf_node.bumpfee(rbfid) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 5e94068930..efa6a199ad 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -19,6 +19,7 @@ class ReceivedByTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_cli() def run_test(self): # Generate block to get out of IBD diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 455e89e310..6f248c9bd3 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -40,6 +40,7 @@ class ListSinceBlockTest(BitcoinTestFramework): def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) blockhash, = self.nodes[2].generate(1) + blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() txs = self.nodes[0].listtransactions() @@ -47,6 +48,7 @@ class ListSinceBlockTest(BitcoinTestFramework): "category": "receive", "amount": 1, "blockhash": blockhash, + "blockheight": blockheight, "confirmations": 1, }) assert_equal( @@ -276,7 +278,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.sync_all() # gettransaction should work for txid1 - self.nodes[0].gettransaction(txid1) + tx1 = self.nodes[0].gettransaction(txid1) + assert_equal(tx1['blockheight'], self.nodes[0].getblockheader(tx1['blockhash'])['height']) # listsinceblock(lastblockhash) should now include txid1 in transactions # as well as in removed diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 997d6e702c..8c44a070b8 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -40,14 +40,15 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid}, {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) # mine a block, confirmations should change: - self.nodes[0].generate(1) + blockhash = self.nodes[0].generate(1)[0] + blockheight = self.nodes[0].getblockheader(blockhash)['height'] self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, - {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1}) + {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, - {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1}) + {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) # send-to-self: txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index f19cf924d2..fde99fe496 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -78,7 +78,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), '-help=1', ], - timeout=1, + timeout=10, check=True, stderr=subprocess.PIPE, universal_newlines=True, diff --git a/test/sanitizer_suppressions/lsan b/test/sanitizer_suppressions/lsan index 90a92a5115..d2cb618d4e 100644 --- a/test/sanitizer_suppressions/lsan +++ b/test/sanitizer_suppressions/lsan @@ -1,5 +1,4 @@ # Suppress warnings triggered in dependencies -leak:libcrypto leak:libqminimal leak:libQt5Core leak:libQt5Gui |