diff options
735 files changed, 11641 insertions, 9454 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 0d026748b5..7250d4ad94 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,29 +1,30 @@ version: '{branch}.{build}' skip_tags: true -image: Previous Visual Studio 2019 +image: Visual Studio 2019 configuration: Release platform: x64 clone_depth: 5 environment: PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' - QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' + QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip' + QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3' QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' - VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' - VCPKG_COMMIT_ID: '40230b8e3f6368dcb398d649331be878ca1e9007' + VCPKG_TAG: '2020.11-1' install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq -# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: +# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is: # a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, -# b. Install the missing packages using the vcpkg manifest. +# b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins). +# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed +# as part of the msbuild command using vcpkg mainfests. - ps: | cd c:\tools\vcpkg $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git pull origin master > $null - git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID + git -c advice.detachedHead=false checkout $env:VCPKG_TAG .\bootstrap-vcpkg.bat > $null + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" cd "$env:APPVEYOR_BUILD_FOLDER" before_build: # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: diff --git a/.cirrus.yml b/.cirrus.yml index 237560fc2e..b915cfabc7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,12 +1,5 @@ ### Global defaults -timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out -container: - # https://cirrus-ci.org/faq/#are-there-any-limits - # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel - cpu: 2 - memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers - kvm: true # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093 env: PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j4" @@ -15,11 +8,27 @@ env: CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" -### Global task template - # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -global_task_template: &GLOBAL_TASK_TEMPLATE +base_template: &BASE_TEMPLATE skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution + merge_base_script: + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - bash -c "$PACKAGE_MANAGER_INSTALL git" + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks + +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out + container: + # https://cirrus-ci.org/faq/#are-there-any-limits + # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel + cpu: 2 + memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers + kvm: true # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093 ccache_cache: folder: "/tmp/ccache_dir" depends_built_cache: @@ -28,16 +37,14 @@ global_task_template: &GLOBAL_TASK_TEMPLATE folder: "/tmp/cirrus-ci-build/depends/sdk-sources" depends_releases_cache: folder: "/tmp/cirrus-ci-build/releases" - merge_base_script: - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts ci_script: - ./ci/test_run_all.sh +compute_credits_template: &CREDITS_TEMPLATE + # https://cirrus-ci.org/pricing/#compute-credits + # Only use credits for pull requests to the main repo + use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" + #task: # name: "Windows" # windows_container: @@ -55,7 +62,19 @@ global_task_template: &GLOBAL_TASK_TEMPLATE # - choco install python --version=3.7.7 -y task: - name: 'ARM [GOAL: install] [buster] [unit tests, no functional tests]' + name: 'lint [bionic]' + << : *BASE_TEMPLATE + container: + image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md + cpu: 1 + memory: 1G + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + lint_script: + - ./ci/lint_run_all.sh + +task: + name: 'ARM [unit tests, no functional tests] [buster]' << : *GLOBAL_TASK_TEMPLATE container: image: debian:buster @@ -63,7 +82,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_arm.sh" task: - name: 'Win64 [GOAL: deploy] [unit tests, no gui, no boost::process, no functional tests]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [bionic]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic @@ -71,7 +90,18 @@ task: FILE_ENV: "./ci/test/00_setup_env_win64.sh" task: - name: 'x86_64 Linux [GOAL: install] [bionic] [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]' + name: '32-bit + dash [gui] [CentOS 8]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: centos:8 + env: + PACKAGE_MANAGER_INSTALL: "yum install -y" + FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" + +task: + name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]' + # For faster CI feedback, immediately schedule a task that compiles most modules + << : *CREDITS_TEMPLATE << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic @@ -79,18 +109,18 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no gui]' + name: '[depends, sanitizers: thread (TSan), no gui] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal - cpu: 4 # Double CPU and Memory to avoid timeout - memory: 16G + cpu: 4 # Double CPU and increase Memory to avoid timeout + memory: 24G env: MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: memory (MSan)]' + name: '[depends, sanitizers: memory (MSan)] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -98,7 +128,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' + name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -106,7 +136,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' + name: '[no depends, sanitizers: fuzzer,address,undefined] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -114,7 +144,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [multiprocess]' + name: '[multiprocess] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -122,7 +152,15 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" task: - name: 'macOS 10.12 [GOAL: deploy] [no functional tests]' + name: '[no wallet] [bionic]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:bionic + env: + FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" + +task: + name: 'macOS 10.14 [gui, no tests] [bionic]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic @@ -130,9 +168,9 @@ task: FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: - name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' + name: 'macOS 10.15 native [gui] [no depends]' macos_brew_addon_script: - - brew install boost libevent berkeley-db4 qt miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt + - brew install boost libevent berkeley-db4 qt miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt << : *GLOBAL_TASK_TEMPLATE osx_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) diff --git a/.fuzzbuzz.yml b/.fuzzbuzz.yml index d44ac27eb9..be9a1cd4e1 100644 --- a/.fuzzbuzz.yml +++ b/.fuzzbuzz.yml @@ -5,9 +5,9 @@ environment: - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1 setup: - sudo apt-get update - - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common + - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libqt5core5a libqt5dbus5 libqt5gui5 libtool llvm llvm-dev llvm-runtime pkg-config qttools5-dev qttools5-dev-tools software-properties-common - ./autogen.sh - - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined + - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined --enable-danger-fuzz-link-all - make - git clone https://github.com/bitcoin-core/qa-assets auto_targets: diff --git a/.gitignore b/.gitignore index 5726b18928..010586ee5a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ src/bitcoin-cli src/bitcoin-gui src/bitcoin-node src/bitcoin-tx +src/bitcoin-util src/bitcoin-wallet src/test/fuzz/* !src/test/fuzz/*.* @@ -76,6 +77,7 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg +*.iso *.json.h *.raw.h diff --git a/.python-version b/.python-version index c49282585a..8b7b0b52e5 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.5.6 +3.6.12 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 656eed9871..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Travis caches can be manually removed if necessary. This is one of the very -# few manual operations that is possible with Travis, and it can be done by a -# Bitcoin Core GitHub member via the Travis web interface [0]. -# -# Travis CI uploads the cache after the script phase of the build [1]. -# However, the build is terminated without saving the cache if it takes over -# 50 minutes [2]. Thus, if we spent too much time in early build stages, fail -# with an error and save the cache. -# -# [0] https://travis-ci.org/bitcoin/bitcoin/caches -# [1] https://docs.travis-ci.com/user/caching/#build-phases -# [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts - -version: ~> 1.0 - -dist: bionic -os: linux -language: minimal -arch: amd64 -cache: - directories: - - $TRAVIS_BUILD_DIR/depends/built - - $TRAVIS_BUILD_DIR/depends/sdk-sources - - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - - $TRAVIS_BUILD_DIR/releases/$HOST -stages: - - lint - - test -env: - global: - - CI_RETRY_EXE="travis_retry" - - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." -before_install: - - set -o errexit; source ./ci/test/00_setup_env.sh - - set -o errexit; source ./ci/test/03_before_install.sh -install: - - set -o errexit; source ./ci/test/04_install.sh -before_script: - # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 - - for i in {1..4}; do echo "$(sleep 500)" ; done & - - set -o errexit; source ./ci/test/05_before_script.sh &> "/dev/null" -script: - - export CONTINUE=1 - - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - - if [[ $SECONDS -gt 50*60-$EXPECTED_TESTS_DURATION_IN_SECONDS ]]; then export CONTINUE=0; fi - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi -after_script: - - echo $TRAVIS_COMMIT_RANGE -jobs: - include: - - - stage: lint - name: 'lint' - env: - cache: pip - language: python - python: '3.5' # Oldest supported version according to doc/dependencies.md - install: - - set -o errexit; source ./ci/lint/04_install.sh - before_script: - - set -o errexit; source ./ci/lint/05_before_script.sh - script: - - set -o errexit; source ./ci/lint/06_script.sh - - - stage: test - name: '32-bit + dash [GOAL: install] [CentOS 7] [gui]' - env: >- - FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" - - - stage: test - name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2020 The Bitcoin Core developers -Copyright (c) 2009-2020 Bitcoin Developers +Copyright (c) 2009-2021 The Bitcoin Core developers +Copyright (c) 2009-2021 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index c8af4228f3..c554f630ae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,7 @@ if ENABLE_MAN SUBDIRS += doc/man endif .PHONY: deploy FORCE +.INTERMEDIATE: $(OSX_TEMP_ISO) $(COVERAGE_INFO) export PYTHONPATH @@ -24,7 +25,10 @@ BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) +BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT) BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) +BITCOIN_NODE_BIN=$(top_builddir)/src/$(BITCOIN_MP_NODE_NAME)$(EXEEXT) +BITCOIN_GUI_BIN=$(top_builddir)/src/$(BITCOIN_MP_GUI_NAME)$(EXEEXT) BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win64-setup$(EXEEXT) empty := @@ -33,15 +37,13 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg +OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso OSX_BACKGROUND_SVG=background.svg OSX_BACKGROUND_IMAGE=background.tiff OSX_BACKGROUND_IMAGE_DPIS=36 72 -OSX_DSSTORE_GEN=$(top_srcdir)/contrib/macdeploy/custom_dsstore.py OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus -OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed -OSX_QT_TRANSLATIONS = ar,bg,ca,cs,da,de,es,fa,fi,fr,gd,gl,he,hu,it,ja,ko,lt,lv,pl,pt,ru,sk,sl,sv,uk,zh_CN,zh_TW DIST_CONTRIB = \ $(top_srcdir)/contrib/linearize/linearize-data.py \ @@ -52,16 +54,16 @@ DIST_SHARE = \ $(top_srcdir)/share/rpcauth BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ - $(top_srcdir)/contrib/devtools/security-check.py + $(top_srcdir)/contrib/devtools/security-check.py \ + $(top_srcdir)/contrib/devtools/pixie.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/share/pixmaps/nsis-header.bmp \ $(top_srcdir)/share/pixmaps/nsis-wizard.bmp \ $(top_srcdir)/doc/README_windows.txt -OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ +OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ - $(OSX_DSSTORE_GEN) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh @@ -80,6 +82,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_UTIL_BIN) $(top_builddir)/release @test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \ echo error: could not build $@ @echo built $@ @@ -117,7 +120,7 @@ osx_volname: if BUILD_DARWIN $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) - $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -add-qt-tr $(OSX_QT_TRANSLATIONS) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -fancy $(OSX_FANCY_PLIST) -verbose 2 -volname $(OSX_VOLNAME) + $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg $(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ @@ -137,8 +140,11 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt -$(OSX_DMG): $(APP_DIST_EXTRAS) - $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist +$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist + +$(OSX_DMG): $(OSX_TEMP_ISO) + $(DMG) dmg "$<" "$@" dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ @@ -147,11 +153,8 @@ $(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIF $(MKDIR_P) $(@D) $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ -$(APP_DIST_DIR)/.DS_Store: $(OSX_DSSTORE_GEN) - $(PYTHON) $< "$@" "$(OSX_VOLNAME)" - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) -translations-dir=$(QT_TRANSLATION_DIR) -add-qt-tr $(OSX_QT_TRANSLATIONS) -verbose 2 + INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) deploydir: $(APP_DIST_EXTRAS) endif @@ -176,9 +179,18 @@ $(BITCOIN_CLI_BIN): FORCE $(BITCOIN_TX_BIN): FORCE $(MAKE) -C src $(@F) +$(BITCOIN_UTIL_BIN): FORCE + $(MAKE) -C src $(@F) + $(BITCOIN_WALLET_BIN): FORCE $(MAKE) -C src $(@F) +$(BITCOIN_NODE_BIN): FORCE + $(MAKE) -C src $(@F) + +$(BITCOIN_GUI_BIN): FORCE + $(MAKE) -C src $(@F) + if USE_LCOV LCOV_FILTER_PATTERN = \ -p "/usr/local/" \ @@ -326,8 +338,6 @@ EXTRA_DIST += \ CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) -.INTERMEDIATE: $(COVERAGE_INFO) - DISTCHECK_CONFIGURE_FLAGS = --enable-man doc/doxygen/.stamp: doc/Doxyfile FORCE @@ -353,10 +363,13 @@ clean-local: clean-docs test-security-check: if TARGET_DARWIN $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif @@ -56,7 +56,8 @@ There are also [regression and integration tests](/test), written in Python, that are run automatically on the build server. These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py` -The Travis CI system makes sure that every pull request is built for Windows, Linux, and macOS, and that unit/sanity tests are run automatically. +The CI (Continuous Integration) systems make sure that every pull request is built for Windows, Linux, and macOS, +and that unit/sanity tests are run automatically. ### Manual Quality Assurance (QA) Testing diff --git a/CODEOWNERS b/REVIEWERS index 24a80fb35d..fa9a8f525f 100644 --- a/CODEOWNERS +++ b/REVIEWERS @@ -1,20 +1,15 @@ # ============================================================================== -# Bitcoin Core CODEOWNERS +# Bitcoin Core REVIEWERS # ============================================================================== -# Configuration of code ownership and review approvals for the bitcoin/bitcoin -# repo. +# Configuration of automated review requests for the bitcoin/bitcoin repo +# via DrahtBot. -# Order is important; the last matching pattern takes the most precedence. -# More info on how this file works can be found at: -# https://help.github.com/articles/about-codeowners/ +# Order is not important; if a modified file or directory matches a fnmatch, +# the reviewer will be mentioned in a PR comment requesting a review. -# This file is called CODEOWNERS because it is a magic file for GitHub to -# automatically suggest reviewers. In this project's case, the names below -# should be thought of as code reviewers rather than owners. Regular -# contributors are free to add their names to specific directories or files -# provided that they are willing to provide a review when automatically -# assigned. +# Regular contributors are free to add their names to specific directories or +# files provided that they are willing to provide a review. # Absence from this list should not be interpreted as a discouragement to # review a pull request. Peer review is always welcome and is a critical @@ -23,12 +18,12 @@ # Maintainers -# @laanwj -# @sipa # @fanquake # @jonasschnelli +# @laanwj # @marcofalke # @meshcollider +# @sipa # Docs /doc/*[a-zA-Z-].md @harding diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index aa0111e5a2..5fc5b493d3 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -6,7 +6,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ AC_ARG_VAR(BDB_CFLAGS, [C compiler flags for BerkeleyDB, bypasses autodetection]) AC_ARG_VAR(BDB_LIBS, [Linker flags for BerkeleyDB, bypasses autodetection]) - if test "x$BDB_CFLAGS" = "x"; then + if test "x$use_bdb" = "xno"; then + use_bdb=no + elif test "x$BDB_CFLAGS" = "x"; then AC_MSG_CHECKING([for Berkeley DB C++ headers]) BDB_CPPFLAGS= bdbpath=X @@ -44,25 +46,30 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ ],[]) done if test "x$bdbpath" = "xX"; then + use_bdb=no AC_MSG_RESULT([no]) - AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for wallet functionality (--disable-wallet to disable wallet functionality)]) + AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) elif test "x$bdb48path" = "xX"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ - AC_MSG_WARN([Found Berkeley DB other than 4.8; wallets opened by this build will not be portable!]) + AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!]) ],[ - AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable wallets (--with-incompatible-bdb to ignore or --disable-wallet to disable wallet functionality)]) + AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)]) ]) + use_bdb=yes else BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx) bdbpath="${bdb48path}" + use_bdb=yes fi else BDB_CPPFLAGS=${BDB_CFLAGS} fi AC_SUBST(BDB_CPPFLAGS) - if test "x$BDB_LIBS" = "x"; then + if test "x$use_bdb" = "xno"; then + use_bdb=no + elif test "x$BDB_LIBS" = "x"; then # TODO: Ideally this could find the library version and make sure it matches the headers being used for searchlib in db_cxx-4.8 db_cxx db4_cxx; do AC_CHECK_LIB([$searchlib],[main],[ @@ -71,8 +78,12 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ ]) done if test "x$BDB_LIBS" = "x"; then - AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for wallet functionality (--disable-wallet to disable wallet functionality)]) + AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) fi fi - AC_SUBST(BDB_LIBS) + if test "x$use_bdb" != "xno"; then + AC_SUBST(BDB_LIBS) + AC_DEFINE([USE_BDB], [1], [Define if BDB support should be compiled in]) + use_bdb=yes + fi ]) diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 6c7665830b..658dfc5476 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -128,7 +128,6 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QXcbIntegrationPlugin)],[-lqxcb -lxcb-static]) AC_DEFINE(QT_QPA_PLATFORM_XCB, 1, [Define this symbol if the qt platform is xcb]) elif test "x$TARGET_OS" = xdarwin; then - AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) elif test "x$TARGET_OS" = xandroid; then @@ -202,7 +201,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ *darwin*) BITCOIN_QT_CHECK([ MOC_DEFS="${MOC_DEFS} -DQ_OS_MAC" - base_frameworks="-framework Foundation -framework ApplicationServices -framework AppKit" + base_frameworks="-framework Foundation -framework AppKit" AX_CHECK_LINK_FLAG([[$base_frameworks]],[QT_LIBS="$QT_LIBS $base_frameworks"],[AC_MSG_ERROR(could not find base frameworks)]) ]) ;; diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index 17cd31a52e..65ce1ee9da 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -56,7 +56,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206</AdditionalOptions> + <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 9d0b50a0b4..53aead38b5 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -15,13 +15,10 @@ #define CLIENT_VERSION_IS_RELEASE false /* Major version */ -#define CLIENT_VERSION_MAJOR 0 +#define CLIENT_VERSION_MAJOR 21 /* Minor version */ -#define CLIENT_VERSION_MINOR 20 - -/* Build revision */ -#define CLIENT_VERSION_REVISION 99 +#define CLIENT_VERSION_MINOR 99 /* Copyright holder(s) before %s replacement */ #define COPYRIGHT_HOLDERS "The %s developers" @@ -38,6 +35,12 @@ /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 +/* Define to 1 to enable BDB wallet */ +#define USE_BDB 1 + +/* Define to 1 to enable SQLite wallet */ +#define USE_SQLITE 1 + /* Define to 1 to enable ZMQ functions */ #define ENABLE_ZMQ 1 @@ -253,7 +256,7 @@ #define PACKAGE_NAME "Bitcoin Core" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 0.19.99" +#define PACKAGE_STRING "Bitcoin Core 21.99.0" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "bitcoin" @@ -262,7 +265,7 @@ #define PACKAGE_URL "https://bitcoincore.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "0.19.99" +#define PACKAGE_VERSION "21.99.0" /* Define to necessary symbol if this constant uses a non-standard name on your system. */ diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index ae24cb100e..48dfafaee0 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -63,6 +63,10 @@ <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@ENABLE_WALLET_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" + Replace="@USE_BDB_TRUE@" By=""></ReplaceInFile> + <ReplaceInFile FilePath="$(ConfigIniOut)" + Replace="@USE_SQLITE_TRUE@" By=""></ReplaceInFile> + <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@BUILD_BITCOIN_CLI_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@BUILD_BITCOIND_TRUE@" By=""></ReplaceInFile> diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index ed227519ae..9c589bccbc 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -4,8 +4,6 @@ <PropertyGroup Label="Globals"> <VCProjectVersion>16.0</VCProjectVersion> - <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet> - <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet> <UseNativeEnvironment>true</UseNativeEnvironment> </PropertyGroup> @@ -16,6 +14,8 @@ <VcpkgUseStatic>true</VcpkgUseStatic> <VcpkgAutoLink>true</VcpkgAutoLink> <VcpkgConfiguration>$(Configuration)</VcpkgConfiguration> + <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet> + <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet> </PropertyGroup> <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'=='' and !Exists('$(WindowsSdkDir)\DesignTime\CommonConfiguration\Neutral\Windows.props')"> @@ -45,66 +45,46 @@ </ProjectConfiguration> </ItemGroup> - <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> - <LinkIncremental>true</LinkIncremental> - <WholeProgramOptimization>false</WholeProgramOptimization> - <UseDebugLibraries>true</UseDebugLibraries> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <LinkIncremental>false</LinkIncremental> + <UseDebugLibraries>false</UseDebugLibraries> <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> + <GenerateManifest>No</GenerateManifest> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> - <LinkIncremental>false</LinkIncremental> - <WholeProgramOptimization>true</WholeProgramOptimization> - <UseDebugLibraries>false</UseDebugLibraries> + + <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> + <LinkIncremental>true</LinkIncremental> + <UseDebugLibraries>true</UseDebugLibraries> <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> </PropertyGroup> -<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> - <ClCompile> - <Optimization>MaxSpeed</Optimization> - <FunctionLevelLinking>true</FunctionLevelLinking> - <IntrinsicFunctions>true</IntrinsicFunctions> - <SDLCheck>true</SDLCheck> - <RuntimeLibrary>MultiThreaded</RuntimeLibrary> - </ClCompile> - <Link> - <EnableCOMDATFolding>true</EnableCOMDATFolding> - <OptimizeReferences>true</OptimizeReferences> - </Link> - </ItemDefinitionGroup> - - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> +<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> <ClCompile> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <SDLCheck>true</SDLCheck> - <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> - <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> - </ClCompile> - </ItemDefinitionGroup> - - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <ClCompile> - <Optimization>MaxSpeed</Optimization> + <WholeProgramOptimization>false</WholeProgramOptimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <SDLCheck>true</SDLCheck> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <DebugInformationFormat>None</DebugInformationFormat> </ClCompile> <Link> - <EnableCOMDATFolding>true</EnableCOMDATFolding> - <OptimizeReferences>true</OptimizeReferences> + <EnableCOMDATFolding>false</EnableCOMDATFolding> + <OptimizeReferences>false</OptimizeReferences> + <AdditionalOptions>/LTCG:OFF</AdditionalOptions> </Link> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <Optimization>Disabled</Optimization> + <WholeProgramOptimization>false</WholeProgramOptimization> <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> @@ -124,7 +104,6 @@ </ClCompile> <Link> <SubSystem>Console</SubSystem> - <GenerateDebugInformation>true</GenerateDebugInformation> <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> <Lib> diff --git a/build_msvc/common.vcxproj b/build_msvc/common.vcxproj index 4bbcc3767f..270c75e8a7 100644 --- a/build_msvc/common.vcxproj +++ b/build_msvc/common.vcxproj @@ -4,7 +4,7 @@ <Target Name="CopyBuildArtifacts" Condition="'$(ConfigurationType)' != 'StaticLibrary'"> <ItemGroup> <BuildArtifacts Include="$(OutDir)$(TargetName)$(TargetExt)"></BuildArtifacts> - <BuildArtifacts Include="$(OutDir)$(TargetName).pdb"></BuildArtifacts> + <BuildArtifacts Include="$(OutDir)$(TargetName).pdb" Condition="Exists('$(OutDir)$(TargetName).pdb')"></BuildArtifacts> </ItemGroup> <Copy SourceFiles="@(BuildArtifacts)" SkipUnchangedFiles="true" DestinationFolder="..\..\src\" Condition="'$(OutDir)' != ''"></Copy> </Target> diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index 9c8279c72a..613d5c7199 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -8,6 +8,9 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\wallet\bdb.cpp" /> + <ClCompile Include="..\..\src\wallet\salvage.cpp" /> + <ClCompile Include="..\..\src\wallet\sqlite.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 2095c0c321..1ddd62edf2 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -73,7 +73,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206</AdditionalOptions> + <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions> </Link> </ItemDefinitionGroup> @@ -83,7 +83,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalOptions>/ignore:4206</AdditionalOptions> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> <ItemGroup> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 5fdd97dc78..115c92792d 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index fae424051d..a0b579de1e 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -travis_retry sudo apt update && sudo apt install -y clang-format-9 -sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +${CI_RETRY_EXE} apt-get update +${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq +update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 -travis_retry pip3 install codespell==1.17.1 -travis_retry pip3 install flake8==3.8.3 -travis_retry pip3 install yq -travis_retry pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install codespell==2.0.0 +${CI_RETRY_EXE} pip3 install flake8==3.8.3 +${CI_RETRY_EXE} pip3 install yq +${CI_RETRY_EXE} pip3 install mypy==0.781 SHELLCHECK_VERSION=v0.7.1 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/lint/05_before_script.sh b/ci/lint/05_before_script.sh deleted file mode 100755 index 2987812c8e..0000000000 --- a/ci/lint/05_before_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -git fetch --unshallow diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index dc0f9b923b..de89a09d6a 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then - # TRAVIS_BRANCH will be present in a Travis environment. For builds triggered - # by a pull request this is the name of the branch targeted by the pull request. - # https://docs.travis-ci.com/user/environment-variables/ - COMMIT_RANGE="$TRAVIS_BRANCH..HEAD" +GIT_HEAD=$(git rev-parse HEAD) +if [ -n "$CIRRUS_PR" ]; then + COMMIT_RANGE="$CIRRUS_BASE_SHA..$GIT_HEAD" test/lint/commit-script-check.sh $COMMIT_RANGE fi +export COMMIT_RANGE +# This only checks that the trees are pure subtrees, it is not doing a full +# check with -r to not have to fetch all the remotes. test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 test/lint/git-subtree-check.sh src/univalue @@ -23,8 +24,11 @@ test/lint/check-doc.py test/lint/check-rpc-mappings.py . test/lint/lint-all.sh -if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" ] && [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then +if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && + ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi + +echo +git log --no-merges --oneline $COMMIT_RANGE diff --git a/ci/lint_run_all.sh b/ci/lint_run_all.sh new file mode 100755 index 0000000000..7adfe71674 --- /dev/null +++ b/ci/lint_run_all.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -o errexit; source ./ci/test/00_setup_env.sh +set -o errexit; source ./ci/lint/04_install.sh +set -o errexit; source ./ci/lint/06_script.sh diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 610e55c4c3..42783197a9 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -25,4 +25,4 @@ export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --with-boost-process" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index e58003ab19..1ec44a7a1a 100644 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -7,9 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export CONTAINER_NAME=ci_i686_centos_7 -export DOCKER_NAME_TAG=centos:7 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash" +export CONTAINER_NAME=ci_i686_centos_8 +export DOCKER_NAME_TAG=centos:8 +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 dash rsync coreutils" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --with-boost-process" export CONFIG_SHELL="/bin/dash" +export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index e4450a65ce..b0de2ec0bb 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -8,11 +8,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) -export HOST=x86_64-apple-darwin16 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" +export HOST=x86_64-apple-darwin18 +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 7c25a34cfe..274a0d1b7c 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -6,10 +6,10 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin16 +export HOST=x86_64-apple-darwin18 export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 191b8049b0..f682486088 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 export GOAL="install" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index b88ee2b50f..3ce50f816f 100644 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 0a09bfe230..7ff044c020 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet -export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md -export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md +export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md +export PACKAGES="python3-zmq clang-5.0 llvm-5.0" # Use clang-5 to test C++17 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8 --with-boost-process" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-5.0 CXX=clang++-5.0 --with-boost-process" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 8668662299..567145fe47 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can compile our c++17 and run our functional tests in python3 +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c++17 and run our functional tests in python3, see doc/dependencies.md export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" +export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index b14a46562c..182e42ee7d 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,6 +10,5 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" -export TEST_RUNNER_EXTRA="--exclude feature_block" # Low memory on Travis machines, exclude feature_block. export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++' --with-boost-process" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index bfaea13a25..f0c153158b 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 72cc3f63c4..affaaaa1aa 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -14,3 +14,7 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_SECURITY_TESTS="true" export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" + +# Compiler for MinGW-w64 causes false -Wreturn-type warning. +# See https://sourceforge.net/p/mingw-w64/bugs/306/ +export NO_WERROR=1 diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh deleted file mode 100755 index 80806aab75..0000000000 --- a/ci/test/03_before_install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BEGIN_FOLD () { - echo "" - CURRENT_FOLD_NAME=$1 - echo "travis_fold:start:${CURRENT_FOLD_NAME}" -} - -END_FOLD () { - RET=$? - echo "travis_fold:end:${CURRENT_FOLD_NAME}" - if [ $RET != 0 ]; then - echo "${CURRENT_FOLD_NAME} failed with status code ${RET}" - fi -} - diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index db74fe6569..f0ed314d19 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,9 +6,6 @@ export LC_ALL=C.UTF-8 -if [[ $DOCKER_NAME_TAG == centos* ]]; then - export LC_ALL=en_US.utf8 -fi if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi @@ -60,8 +57,8 @@ if [ -n "$DPKG_ADD_ARCH" ]; then fi if [[ $DOCKER_NAME_TAG == centos* ]]; then - ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release - ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES + ${CI_RETRY_EXE} DOCKER_EXEC dnf -y install epel-release + ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install $DOCKER_PACKAGES $PACKAGES elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 42c244c2f5..4644f28a4e 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -40,14 +40,12 @@ if [ -z "$NO_DEPENDS" ]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) - SHELL_OPTS="CONFIG_SHELL=/bin/bash" + SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/bash" else SHELL_OPTS="CONFIG_SHELL=" fi DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then - BEGIN_FOLD previous-versions DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" - END_FOLD fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 17d765b862..7986f7665a 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,33 +6,28 @@ export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -BEGIN_FOLD autogen if [ -n "$CONFIG_SHELL" ]; then DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh" else DOCKER_EXEC ./autogen.sh fi -END_FOLD DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" export P_CI_DIR="${BASE_BUILD_DIR}" -BEGIN_FOLD configure DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) -END_FOLD -BEGIN_FOLD distdir DOCKER_EXEC make distdir VERSION=$HOST -END_FOLD export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" -BEGIN_FOLD configure DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) -END_FOLD set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR @@ -45,12 +40,8 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) -END_FOLD -BEGIN_FOLD cache_stats DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" -END_FOLD diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 7aea21f257..194b14beab 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -7,55 +7,39 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then - BEGIN_FOLD wrap-wine # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" - END_FOLD fi if [ -n "$QEMU_USER_CMD" ]; then - BEGIN_FOLD wrap-qemu # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" - END_FOLD fi if [ -n "$USE_VALGRIND" ]; then - BEGIN_FOLD wrap-valgrind DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" - END_FOLD fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - BEGIN_FOLD unit-tests DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 - END_FOLD fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - BEGIN_FOLD unit-tests-seq DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite - END_FOLD fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - BEGIN_FOLD functional-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast - END_FOLD fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then - BEGIN_FOLD security-tests DOCKER_EXEC make test-security-check - END_FOLD fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - BEGIN_FOLD fuzz-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} - END_FOLD fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index be7d7fcc1f..2cd7c8cec2 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index a1d4bd1952..93b07aab1e 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh -set -o errexit; source ./ci/test/03_before_install.sh set -o errexit; source ./ci/test/04_install.sh set -o errexit; source ./ci/test/05_before_script.sh set -o errexit; source ./ci/test/06_script_a.sh diff --git a/configure.ac b/configure.ac index 9a16b7c880..b64c22450c 100644 --- a/configure.ac +++ b/configure.ac @@ -1,14 +1,13 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 20) -define(_CLIENT_VERSION_REVISION, 99) +define(_CLIENT_VERSION_MAJOR, 21) +define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2020) +define(_COPYRIGHT_YEAR, 2021) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) -AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_REVISION, m4_if(_CLIENT_VERSION_BUILD, [0], [], _CLIENT_VERSION_BUILD))m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) +AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) AC_CONFIG_SRCDIR([src/validation.cpp]) AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) @@ -24,7 +23,11 @@ BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx +BITCOIN_UTIL_NAME=bitcoin-util BITCOIN_WALLET_TOOL_NAME=bitcoin-wallet +dnl Multi Process +BITCOIN_MP_NODE_NAME=bitcoin-node +BITCOIN_MP_GUI_NAME=bitcoin-gui dnl Unless the user specified ARFLAGS, force it to be cr AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to <cr> if not set]) @@ -68,18 +71,8 @@ case $host in ;; esac -AC_ARG_ENABLE([c++17], - [AS_HELP_STRING([--enable-c++17], - [enable compilation in c++17 mode (disabled by default)])], - [use_cxx17=$enableval], - [use_cxx17=no]) - -dnl Require C++11 or C++17 compiler (no GNU extensions) -if test "x$use_cxx17" = xyes -o "x$enable_fuzz" = xyes ; then - AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) -else - AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) -fi +dnl Require C++17 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC @@ -107,14 +100,13 @@ AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_TOOL(LLVM_COV, llvm-cov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python]) +dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(READELF, readelf) AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_PATH_PROG(DOXYGEN, doxygen) @@ -126,7 +118,7 @@ AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], [disable wallet (enabled by default)])], [enable_wallet=$enableval], - [enable_wallet=yes]) + [enable_wallet=auto]) AC_ARG_WITH([sqlite], [AS_HELP_STRING([--with-sqlite=yes|no|auto], @@ -134,6 +126,12 @@ AC_ARG_WITH([sqlite], [use_sqlite=$withval], [use_sqlite=auto]) +AC_ARG_WITH([bdb], + [AS_HELP_STRING([--without-bdb], + [disable bdb wallet support (default is enabled if wallet is enabled)])], + [use_bdb=$withval], + [use_bdb=auto]) + AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], [enable UPNP (default is yes if libminiupnpc is found)])], @@ -146,6 +144,18 @@ AC_ARG_ENABLE([upnp-default], [use_upnp_default=$enableval], [use_upnp_default=no]) +AC_ARG_WITH([natpmp], + [AS_HELP_STRING([--with-natpmp], + [enable NAT-PMP (default is yes if libnatpmp is found)])], + [use_natpmp=$withval], + [use_natpmp=auto]) + +AC_ARG_ENABLE([natpmp-default], + [AS_HELP_STRING([--enable-natpmp-default], + [if NAT-PMP is enabled, turn it on at startup (default is no)])], + [use_natpmp_default=$enableval], + [use_natpmp_default=no]) + AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], @@ -172,6 +182,12 @@ AC_ARG_ENABLE([fuzz], [enable_fuzz=$enableval], [enable_fuzz=no]) +AC_ARG_ENABLE([danger_fuzz_link_all], + AS_HELP_STRING([--enable-danger-fuzz-link-all], + [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]), + [enable_danger_fuzz_link_all=$enableval], + [enable_danger_fuzz_link_all=no]) + AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], @@ -353,6 +369,7 @@ if test "x$enable_debug" = xyes; then AX_CHECK_PREPROC_FLAG([-DDEBUG],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-ftrapv],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"],,[[$CXXFLAG_WERROR]]) fi @@ -419,6 +436,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-member-function],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) @@ -554,7 +572,7 @@ CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], - [build bitcoin-cli bitcoin-tx bitcoin-wallet (default=yes)])], + [build bitcoin-cli bitcoin-tx bitcoin-util bitcoin-wallet (default=yes)])], [build_bitcoin_utils=$withval], [build_bitcoin_utils=yes]) @@ -576,6 +594,12 @@ AC_ARG_ENABLE([util-wallet], [build_bitcoin_wallet=$enableval], [build_bitcoin_wallet=$build_bitcoin_utils]) +AC_ARG_ENABLE([util-util], + [AS_HELP_STRING([--enable-util-util], + [build bitcoin-util])], + [build_bitcoin_util=$enableval], + [build_bitcoin_util=$build_bitcoin_utils]) + AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], @@ -646,16 +670,19 @@ case $host in dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) - qt5_prefix=$($BREW --prefix qt5 2>/dev/null) - if test x$bdb_prefix != x && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then + bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. BDB_CFLAGS="-I$bdb_prefix/include" BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test x$qt5_prefix != x; then - PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" - export PKG_CONFIG_PATH + + if test "x$use_sqlite" != xno && $BREW list --versions sqlite3 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" + fi + + if $BREW list --versions qt5 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi fi else @@ -667,7 +694,8 @@ case $host in AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) + AC_PATH_PROGS([DMG], [dmg], dmg) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) @@ -744,6 +772,9 @@ if test x$use_lcov_branch != xno; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi +dnl Check for __int128 +AC_CHECK_TYPES([__int128]) + dnl Check for endianness AC_C_BIGENDIAN @@ -1114,18 +1145,20 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT(yes); HAVE_STRONG_GETAUXVAL=1 ], + [ AC_MSG_RESULT(yes); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE(HAVE_STRONG_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval)]) ], [ AC_MSG_RESULT(no); HAVE_STRONG_GETAUXVAL=0 ] ) AC_MSG_CHECKING(for weak getauxval support in the compiler) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #ifdef __linux__ unsigned long getauxval(unsigned long type) __attribute__((weak)); #define AT_HWCAP 16 + #endif ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT(yes); HAVE_WEAK_GETAUXVAL=1 ], + [ AC_MSG_RESULT(yes); HAVE_WEAK_GETAUXVAL=1; AC_DEFINE(HAVE_WEAK_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval (weak linking)]) ], [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] ) @@ -1183,6 +1216,7 @@ if test "x$enable_fuzz" = "xyes"; then build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no + build_bitcoin_util=no build_bitcoin_wallet=no build_bitcoind=no build_bitcoin_libs=no @@ -1192,8 +1226,11 @@ if test "x$enable_fuzz" = "xyes"; then enable_wallet=no use_bench=no use_upnp=no + use_natpmp=no use_zmq=no + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) + AC_MSG_CHECKING([whether main function is needed]) AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], @@ -1248,6 +1285,14 @@ if test x$enable_wallet != xno; then fi fi AC_MSG_RESULT([$use_sqlite]) + + dnl Disable wallet if both --without-bdb and --without-sqlite + if test "x$use_bdb$use_sqlite" = "xnono"; then + if test "x$enable_wallet" = "xyes"; then + AC_MSG_ERROR([wallet functionality requested but no BDB or SQLite support available.]) + fi + enable_wallet=no + fi fi dnl Check for libminiupnpc (optional) @@ -1279,6 +1324,13 @@ if test x$have_miniupnpc != xno; then fi fi +dnl Check for libnatpmp (optional). +if test "x$use_natpmp" != xno; then + AC_CHECK_HEADERS([natpmp.h], + [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], + [have_natpmp=no]) +fi + if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then use_boost=no else @@ -1389,7 +1441,7 @@ fi dnl univalue check need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then +if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then need_bundled_univalue=no else if test x$system_univalue != xno; then @@ -1472,6 +1524,10 @@ AC_MSG_CHECKING([whether to build bitcoin-wallet]) AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test x$build_bitcoin_wallet = xyes]) AC_MSG_RESULT($build_bitcoin_wallet) +AC_MSG_CHECKING([whether to build bitcoin-util]) +AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test x$build_bitcoin_util = xyes]) +AC_MSG_RESULT($build_bitcoin_util) + AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) if test x$build_bitcoin_libs = xyes; then @@ -1503,6 +1559,7 @@ AC_MSG_CHECKING([if wallet should be enabled]) if test x$enable_wallet != xno; then AC_MSG_RESULT(yes) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) + enable_wallet=yes else AC_MSG_RESULT(no) @@ -1536,6 +1593,31 @@ else fi fi +dnl Enable NAT-PMP support. +AC_MSG_CHECKING([whether to build with support for NAT-PMP]) +if test "x$have_natpmp" = xno; then + if test "x$use_natpmp" = xyes; then + AC_MSG_ERROR([NAT-PMP requested but cannot be built. Use --without-natpmp]) + fi + AC_MSG_RESULT([no]) + use_natpmp=no +else + if test "x$use_natpmp" != xno; then + AC_MSG_RESULT([yes]) + AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) + use_natpmp=yes + natpmp_setting=0 + if test "x$use_natpmp_default" != xno; then + use_natpmp_default=yes + natpmp_setting=1 + fi + AC_MSG_RESULT($use_natpmp_default) + AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + else + AC_MSG_RESULT([no]) + fi +fi + dnl these are only used when qt is enabled BUILD_TEST_QT="" if test x$bitcoin_enable_qt != xno; then @@ -1578,7 +1660,11 @@ AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) if test x$use_tests = xyes; then - AC_MSG_RESULT([yes]) + if test "x$enable_fuzz" = "xyes"; then + AC_MSG_RESULT([no, because fuzzing is enabled]) + else + AC_MSG_RESULT([yes]) + fi BUILD_TEST="yes" else AC_MSG_RESULT([no]) @@ -1602,8 +1688,10 @@ AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) +AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) +AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) @@ -1622,7 +1710,6 @@ AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) -AC_DEFINE(CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION, [Build revision]) AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build]) AC_DEFINE(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE, [Version is release]) AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) @@ -1632,7 +1719,6 @@ define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) -AC_SUBST(CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION) AC_SUBST(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD) AC_SUBST(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE) AC_SUBST(COPYRIGHT_YEAR, _COPYRIGHT_YEAR) @@ -1643,7 +1729,10 @@ AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) +AC_SUBST(BITCOIN_UTIL_NAME) AC_SUBST(BITCOIN_WALLET_TOOL_NAME) +AC_SUBST(BITCOIN_MP_NODE_NAME) +AC_SUBST(BITCOIN_MP_GUI_NAME) AC_SUBST(RELDFLAGS) AC_SUBST(DEBUG_CPPFLAGS) @@ -1668,6 +1757,7 @@ AC_SUBST(SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) +AC_SUBST(USE_BDB) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) @@ -1675,6 +1765,7 @@ AC_SUBST(SQLITE_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) +AC_SUBST(NATPMP_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) @@ -1691,7 +1782,9 @@ AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-check.py]) +AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) +AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) @@ -1745,18 +1838,22 @@ echo " multiprocess = $build_multiprocess" echo " with wallet = $enable_wallet" if test "x$enable_wallet" != "xno"; then echo " with sqlite = $use_sqlite" + echo " with bdb = $use_bdb" fi echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" -echo " with test = $use_tests" -if test x$use_tests != xno; then +if test x$enable_fuzz == xno; then + echo " with test = $use_tests" +else + echo " with test = not building test_bitcoin because fuzzing is enabled" echo " with fuzz = $enable_fuzz" fi echo " with bench = $use_bench" echo " with upnp = $use_upnp" +echo " with natpmp = $use_natpmp" echo " use asm = $use_asm" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 581fe712e9..a18c5bccc5 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2020, Bitcoin Core Developers +Copyright: 2009-2021, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index bc5f09a3e2..b1d9f2b7db 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import sys import re +from typing import Dict, List, Set MAPPING = { 'core_read.cpp': 'core_io.cpp', @@ -32,7 +33,7 @@ def module_name(path): return None files = dict() -deps = dict() +deps: Dict[str, Set[str]] = dict() RE = re.compile("^#include <(.*)>") @@ -59,12 +60,12 @@ for arg in sorted(files.keys()): deps[module].add(included_module) # Loop to find the shortest (remaining) circular dependency -have_cycle = False +have_cycle: bool = False while True: shortest_cycle = None for module in sorted(deps.keys()): # Build the transitive closure of dependencies of module - closure = dict() + closure: Dict[str, List[str]] = dict() for dep in deps[module]: closure[dep] = [] while True: diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index aa65953d83..b7bf76ce77 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,10 +14,27 @@ BITCOIND=${BITCOIND:-$BINDIR/bitcoind} BITCOINCLI=${BITCOINCLI:-$BINDIR/bitcoin-cli} BITCOINTX=${BITCOINTX:-$BINDIR/bitcoin-tx} WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} +BITCOINUTIL=${BITCOINQT:-$BINDIR/bitcoin-util} BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} [ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 +# Don't allow man pages to be generated for binaries built from a dirty tree +DIRTY="" +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do + VERSION_OUTPUT=$($cmd --version) + if [[ $VERSION_OUTPUT == *"dirty"* ]]; then + DIRTY="${DIRTY}${cmd}\n" + fi +done +if [ -n "$DIRTY" ] +then + echo -e "WARNING: the following binaries were built from a dirty tree:\n" + echo -e $DIRTY + echo "man pages generated from dirty binaries should NOT be committed." + echo "To properly generate man pages, please commit your changes to the above binaries, rebuild them, then run this script again." +fi + # The autodetected version git tag can screw up manpage output a little bit read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" @@ -27,7 +44,7 @@ read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ prin echo "[COPYRIGHT]" > footer.h2m $BITCOIND --version | sed -n '1!p' >> footer.h2m -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINQT; do +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do cmdname="${cmd##*/}" help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py new file mode 100644 index 0000000000..8cf06a799a --- /dev/null +++ b/contrib/devtools/pixie.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Compact, self-contained ELF implementation for bitcoin-core security checks. +''' +import struct +import types +from typing import Dict, List, Optional, Union, Tuple + +# you can find all these values in elf.h +EI_NIDENT = 16 + +# Byte indices in e_ident +EI_CLASS = 4 # ELFCLASSxx +EI_DATA = 5 # ELFDATAxxxx + +ELFCLASS32 = 1 # 32-bit +ELFCLASS64 = 2 # 64-bit + +ELFDATA2LSB = 1 # little endian +ELFDATA2MSB = 2 # big endian + +# relevant values for e_machine +EM_386 = 3 +EM_PPC64 = 21 +EM_ARM = 40 +EM_AARCH64 = 183 +EM_X86_64 = 62 +EM_RISCV = 243 + +# relevant values for e_type +ET_DYN = 3 + +# relevant values for sh_type +SHT_PROGBITS = 1 +SHT_STRTAB = 3 +SHT_DYNAMIC = 6 +SHT_DYNSYM = 11 +SHT_GNU_verneed = 0x6ffffffe +SHT_GNU_versym = 0x6fffffff + +# relevant values for p_type +PT_LOAD = 1 +PT_GNU_STACK = 0x6474e551 +PT_GNU_RELRO = 0x6474e552 + +# relevant values for p_flags +PF_X = (1 << 0) +PF_W = (1 << 1) +PF_R = (1 << 2) + +# relevant values for d_tag +DT_NEEDED = 1 +DT_FLAGS = 30 + +# relevant values of `d_un.d_val' in the DT_FLAGS entry +DF_BIND_NOW = 0x00000008 + +# relevant d_tags with string payload +STRING_TAGS = {DT_NEEDED} + +# rrlevant values for ST_BIND subfield of st_info (symbol binding) +STB_LOCAL = 0 + +class ELFRecord(types.SimpleNamespace): + '''Unified parsing for ELF records.''' + def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: + hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] + if total_size is not None and hdr_struct.size > total_size: + raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') + for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): + setattr(self, field, value) + +def BiStruct(chars: str) -> Dict[int, struct.Struct]: + '''Compile a struct parser for both endians.''' + return { + ELFDATA2LSB: struct.Struct('<' + chars), + ELFDATA2MSB: struct.Struct('>' + chars), + } + +class ELFHeader(ELFRecord): + FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] + STRUCT = { + ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), + ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), + } + + def __init__(self, data: bytes, offset: int) -> None: + self.e_ident = data[offset:offset + EI_NIDENT] + if self.e_ident[0:4] != b'\x7fELF': + raise ValueError('invalid ELF magic') + self.ei_class = self.e_ident[EI_CLASS] + self.ei_data = self.e_ident[EI_DATA] + + super().__init__(data, offset + EI_NIDENT, self, None) + + def __repr__(self) -> str: + return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' + +class Section(ELFRecord): + name: Optional[bytes] = None + FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] + STRUCT = { + ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), + ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, eh.e_shentsize) + self._data = data + + def __repr__(self) -> str: + return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' + + def contents(self) -> bytes: + '''Return section contents.''' + return self._data[self.sh_offset:self.sh_offset + self.sh_size] + +class ProgramHeader(ELFRecord): + STRUCT = { + # different ELF classes have the same fields, but in a different order to optimize space versus alignment + ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), + ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, eh.e_phentsize) + + def __repr__(self) -> str: + return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' + +class Symbol(ELFRecord): + STRUCT = { + # different ELF classes have the same fields, but in a different order to optimize space versus alignment + ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), + ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: + super().__init__(data, offset, eh, symtab.sh_entsize) + self.name = _lookup_string(strings, self.st_name) + self.version = version + + def __repr__(self) -> str: + return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' + + @property + def is_import(self) -> bool: + '''Returns whether the symbol is an imported symbol.''' + return self.st_bind != STB_LOCAL and self.st_shndx == 0 + + @property + def is_export(self) -> bool: + '''Returns whether the symbol is an exported symbol.''' + return self.st_bind != STB_LOCAL and self.st_shndx != 0 + + @property + def st_bind(self) -> int: + '''Returns STB_*.''' + return self.st_info >> 4 + +class Verneed(ELFRecord): + DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) + STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: + super().__init__(data, offset, eh, None) + + def __repr__(self) -> str: + return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' + +class Vernaux(ELFRecord): + DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) + STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: + super().__init__(data, offset, eh, None) + self.name = _lookup_string(strings, self.vna_name) + + def __repr__(self) -> str: + return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' + +class DynTag(ELFRecord): + STRUCT = { + ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), + ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), + } + + def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: + super().__init__(data, offset, eh, section.sh_entsize) + + def __repr__(self) -> str: + return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' + +def _lookup_string(data: bytes, index: int) -> bytes: + '''Look up string by offset in ELF string table.''' + endx = data.find(b'\x00', index) + assert endx != -1 + return data[index:endx] + +VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section +def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: + '''Parse symbol table, return a list of symbols.''' + data = section.contents() + symbols = [] + versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) + for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): + symbols.append(Symbol(data, ofs, eh, section, strings, version)) + return symbols + +def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: + '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' + data = section.contents() + ofs = 0 + result = {} + while True: + verneed = Verneed(data, ofs, eh) + aofs = verneed.vn_aux + while True: + vernaux = Vernaux(data, aofs, eh, strings) + result[vernaux.vna_other] = vernaux.name + if not vernaux.vna_next: + break + aofs += vernaux.vna_next + + if not verneed.vn_next: + break + ofs += verneed.vn_next + + return result + +def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: + '''Parse dynamic tags. Return array of tuples.''' + data = section.contents() + ofs = 0 + result = [] + for ofs in range(0, len(data), section.sh_entsize): + tag = DynTag(data, ofs, eh, section) + val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val + result.append((tag.d_tag, val)) + + return result + +class ELFFile: + sections: List[Section] + program_headers: List[ProgramHeader] + dyn_symbols: List[Symbol] + dyn_tags: List[Tuple[int, Union[bytes, int]]] + + def __init__(self, data: bytes) -> None: + self.data = data + self.hdr = ELFHeader(self.data, 0) + self._load_sections() + self._load_program_headers() + self._load_dyn_symbols() + self._load_dyn_tags() + self._section_to_segment_mapping() + + def _load_sections(self) -> None: + self.sections = [] + for idx in range(self.hdr.e_shnum): + offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize + self.sections.append(Section(self.data, offset, self.hdr)) + + shstr = self.sections[self.hdr.e_shstrndx].contents() + for section in self.sections: + section.name = _lookup_string(shstr, section.sh_name) + + def _load_program_headers(self) -> None: + self.program_headers = [] + for idx in range(self.hdr.e_phnum): + offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize + self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) + + def _load_dyn_symbols(self) -> None: + # first, load 'verneed' section + verneed = None + for section in self.sections: + if section.sh_type == SHT_GNU_verneed: + strtab = self.sections[section.sh_link].contents() # associated string table + assert verneed is None # only one section of this kind please + verneed = _parse_verneed(section, strtab, self.hdr) + assert verneed is not None + + # then, correlate GNU versym sections with dynamic symbol sections + versym = {} + for section in self.sections: + if section.sh_type == SHT_GNU_versym: + versym[section.sh_link] = section + + # finally, load dynsym sections + self.dyn_symbols = [] + for idx, section in enumerate(self.sections): + if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables + strtab_data = self.sections[section.sh_link].contents() # associated string table + versym_data = versym[idx].contents() # associated symbol version table + self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) + + def _load_dyn_tags(self) -> None: + self.dyn_tags = [] + for idx, section in enumerate(self.sections): + if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables + strtab = self.sections[section.sh_link].contents() # associated string table + self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) + + def _section_to_segment_mapping(self) -> None: + for ph in self.program_headers: + ph.sections = [] + for section in self.sections: + if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): + ph.sections.append(section) + + def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: + '''Return the values of all dyn tags with the specified tag.''' + return [val for (tag, val) in self.dyn_tags if tag == tag_in] + + +def load(filename: str) -> ELFFile: + with open(filename, 'rb') as f: + data = f.read() + return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 02615edb54..7b09c42fde 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -6,15 +6,15 @@ Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `readelf` (for ELF), `objdump` (for PE) and `otool` (for MACHO). +Needs `objdump` (for PE) and `otool` (for MACHO). ''' import subprocess import sys import os - from typing import List, Optional -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +import pixie + OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') @@ -26,75 +26,20 @@ def check_ELF_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - stdout = run_command([READELF_CMD, '-h', '-W', executable]) - - ok = False - for line in stdout.splitlines(): - tokens = line.split() - if len(line)>=2 and tokens[0] == 'Type:' and tokens[1] == 'DYN': - ok = True - return ok - -def get_ELF_program_headers(executable): - '''Return type and flags for ELF program headers''' - stdout = run_command([READELF_CMD, '-l', '-W', executable]) - - in_headers = False - headers = [] - for line in stdout.splitlines(): - if line.startswith('Program Headers:'): - in_headers = True - count = 0 - if line == '': - in_headers = False - if in_headers: - if count == 1: # header line - header = [x.strip() for x in line.split()] - ofs_typ = header.index('Type') - ofs_flags = header.index('Flg') - # assert readelf output is what we expect - if ofs_typ == -1 or ofs_flags == -1: - raise ValueError('Cannot parse elfread -lW output') - elif count > 1: - splitline = [x.strip() for x in line.split()] - typ = splitline[ofs_typ] - if not typ.startswith('[R'): # skip [Requesting ...] - splitline = [x.strip() for x in line.split()] - flags = splitline[ofs_flags] - # check for 'R', ' E' - if splitline[ofs_flags + 1] == 'E': - flags += ' E' - headers.append((typ, flags, [])) - count += 1 - - if line.startswith(' Section to Segment mapping:'): - in_mapping = True - count = 0 - if line == '': - in_mapping = False - if in_mapping: - if count == 1: # header line - ofs_segment = line.find('Segment') - ofs_sections = line.find('Sections...') - if ofs_segment == -1 or ofs_sections == -1: - raise ValueError('Cannot parse elfread -lW output') - elif count > 1: - segment = int(line[ofs_segment:ofs_sections].strip()) - sections = line[ofs_sections:].strip().split() - headers[segment][2].extend(sections) - count += 1 - return headers + elf = pixie.load(executable) + return elf.hdr.e_type == pixie.ET_DYN def check_ELF_NX(executable) -> bool: ''' Check that no sections are writable and executable (including the stack) ''' + elf = pixie.load(executable) have_wx = False have_gnu_stack = False - for (typ, flags, _) in get_ELF_program_headers(executable): - if typ == 'GNU_STACK': + for ph in elf.program_headers: + if ph.p_type == pixie.PT_GNU_STACK: have_gnu_stack = True - if 'W' in flags and 'E' in flags: # section is both writable and executable + if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable have_wx = True return have_gnu_stack and not have_wx @@ -104,35 +49,34 @@ def check_ELF_RELRO(executable) -> bool: GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' + elf = pixie.load(executable) have_gnu_relro = False - for (typ, flags, _) in get_ELF_program_headers(executable): - # Note: not checking flags == 'R': here as linkers set the permission differently + for ph in elf.program_headers: + # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if typ == 'GNU_RELRO': + if ph.p_type == pixie.PT_GNU_RELRO: have_gnu_relro = True have_bindnow = False - stdout = run_command([READELF_CMD, '-d', '-W', executable]) - - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]): + for flags in elf.query_dyn_tags(pixie.DT_FLAGS): + assert isinstance(flags, int) + if flags & pixie.DF_BIND_NOW: have_bindnow = True + return have_gnu_relro and have_bindnow def check_ELF_Canary(executable) -> bool: ''' Check for use of stack canary ''' - stdout = run_command([READELF_CMD, '--dyn-syms', '-W', executable]) - + elf = pixie.load(executable) ok = False - for line in stdout.splitlines(): - if '__stack_chk_fail' in line: + for symbol in elf.dyn_symbols: + if symbol.name == b'__stack_chk_fail': ok = True return ok @@ -142,48 +86,55 @@ def check_ELF_separate_code(executable): based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' + elf = pixie.load(executable) + R = pixie.PF_R + W = pixie.PF_W + E = pixie.PF_X EXPECTED_FLAGS = { # Read + execute - '.init': 'R E', - '.plt': 'R E', - '.plt.got': 'R E', - '.plt.sec': 'R E', - '.text': 'R E', - '.fini': 'R E', + b'.init': R | E, + b'.plt': R | E, + b'.plt.got': R | E, + b'.plt.sec': R | E, + b'.text': R | E, + b'.fini': R | E, # Read-only data - '.interp': 'R', - '.note.gnu.property': 'R', - '.note.gnu.build-id': 'R', - '.note.ABI-tag': 'R', - '.gnu.hash': 'R', - '.dynsym': 'R', - '.dynstr': 'R', - '.gnu.version': 'R', - '.gnu.version_r': 'R', - '.rela.dyn': 'R', - '.rela.plt': 'R', - '.rodata': 'R', - '.eh_frame_hdr': 'R', - '.eh_frame': 'R', - '.qtmetadata': 'R', - '.gcc_except_table': 'R', - '.stapsdt.base': 'R', + b'.interp': R, + b'.note.gnu.property': R, + b'.note.gnu.build-id': R, + b'.note.ABI-tag': R, + b'.gnu.hash': R, + b'.dynsym': R, + b'.dynstr': R, + b'.gnu.version': R, + b'.gnu.version_r': R, + b'.rela.dyn': R, + b'.rela.plt': R, + b'.rodata': R, + b'.eh_frame_hdr': R, + b'.eh_frame': R, + b'.qtmetadata': R, + b'.gcc_except_table': R, + b'.stapsdt.base': R, # Writable data - '.init_array': 'RW', - '.fini_array': 'RW', - '.dynamic': 'RW', - '.got': 'RW', - '.data': 'RW', - '.bss': 'RW', + b'.init_array': R | W, + b'.fini_array': R | W, + b'.dynamic': R | W, + b'.got': R | W, + b'.data': R | W, + b'.bss': R | W, } + if elf.hdr.e_machine == pixie.EM_PPC64: + # .plt is RW on ppc64 even with separate-code + EXPECTED_FLAGS[b'.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for (typ, flags, sections) in get_ELF_program_headers(executable): - if typ == 'LOAD': - for section in sections: - assert(section not in flags_per_section) - flags_per_section[section] = flags + for ph in elf.program_headers: + if ph.p_type == pixie.PT_LOAD: + for section in ph.sections: + assert(section.name not in flags_per_section) + flags_per_section[section.name] = ph.p_flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): @@ -236,7 +187,7 @@ def check_PE_NX(executable) -> bool: def get_MACHO_executable_flags(executable) -> List[str]: stdout = run_command([OTOOL_CMD, '-vh', executable]) - flags = [] + flags: List[str] = [] for line in stdout.splitlines(): tokens = line.split() # filter first two header lines diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 6949cb7ced..b30ed62521 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -11,10 +11,11 @@ Example usage: find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess -import re import sys import os -from typing import List, Optional, Tuple +from typing import List, Optional + +import pixie # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # @@ -50,7 +51,6 @@ IGNORE_EXPORTS = { '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } -READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') @@ -68,6 +68,8 @@ ELF_ALLOWED_LIBRARIES = { 'ld-linux.so.2', # 32-bit dynamic linker 'ld-linux-aarch64.so.1', # 64-bit ARM dynamic linker 'ld-linux-armhf.so.3', # 32-bit ARM dynamic linker +'ld64.so.1', # POWER64 ABIv1 dynamic linker +'ld64.so.2', # POWER64 ABIv2 dynamic linker 'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker # bitcoin-qt only 'libxcb.so.1', # part of X11 @@ -76,11 +78,12 @@ ELF_ALLOWED_LIBRARIES = { 'libdl.so.2' # programming interface to dynamic linker } ARCH_MIN_GLIBC_VER = { -'80386': (2,1), -'X86-64': (2,2,5), -'ARM': (2,4), -'AArch64':(2,17), -'RISC-V': (2,27) +pixie.EM_386: (2,1), +pixie.EM_X86_64: (2,2,5), +pixie.EM_ARM: (2,4), +pixie.EM_AARCH64:(2,17), +pixie.EM_PPC64: (2,17), +pixie.EM_RISCV: (2,27) } MACHO_ALLOWED_LIBRARIES = { @@ -140,29 +143,6 @@ class CPPFilt(object): self.proc.stdout.close() self.proc.wait() -def read_symbols(executable, imports=True) -> List[Tuple[str, str, str]]: - ''' - Parse an ELF executable and return a list of (symbol,version, arch) tuples - for dynamic, imported symbols. - ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Could not read symbols for {}: {}'.format(executable, stderr.strip())) - syms = [] - for line in stdout.splitlines(): - line = line.split() - if 'Machine:' in line: - arch = line[-1] - if len(line)>7 and re.match('[0-9]+:$', line[0]): - (sym, _, version) = line[7].partition('@') - is_import = line[6] == 'UND' - if version.startswith('@'): - version = version[1:] - if is_import == imports: - syms.append((sym, version, arch)) - return syms - def check_version(max_versions, version, arch) -> bool: if '_' in version: (lib, _, ver) = version.rpartition('_') @@ -174,46 +154,42 @@ def check_version(max_versions, version, arch) -> bool: return False return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] -def elf_read_libraries(filename) -> List[str]: - p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>2 and tokens[1] == '(NEEDED)': - match = re.match(r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) - if match: - libraries.append(match.group(1)) - else: - raise ValueError('Unparseable (NEEDED) specification') - return libraries - def check_imported_symbols(filename) -> bool: + elf = pixie.load(filename) cppfilt = CPPFilt() - ok = True - for sym, version, arch in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version, arch): + ok: bool = True + + for symbol in elf.dyn_symbols: + if not symbol.is_import: + continue + sym = symbol.name.decode() + version = symbol.version.decode() if symbol.version is not None else None + if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) ok = False return ok def check_exported_symbols(filename) -> bool: + elf = pixie.load(filename) cppfilt = CPPFilt() - ok = True - for sym,version,arch in read_symbols(filename, False): - if arch == 'RISC-V' or sym in IGNORE_EXPORTS: + ok: bool = True + for symbol in elf.dyn_symbols: + if not symbol.is_export: + continue + sym = symbol.name.decode() + if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: continue print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) ok = False return ok def check_ELF_libraries(filename) -> bool: - ok = True - for library_name in elf_read_libraries(filename): - if library_name not in ELF_ALLOWED_LIBRARIES: - print('{}: NEEDED library {} is not allowed'.format(filename, library_name)) + ok: bool = True + elf = pixie.load(filename) + for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): + assert(isinstance(library_name, bytes)) + if library_name.decode() not in ELF_ALLOWED_LIBRARIES: + print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) ok = False return ok @@ -231,7 +207,7 @@ def macho_read_libraries(filename) -> List[str]: return libraries def check_MACHO_libraries(filename) -> bool: - ok = True + ok: bool = True for dylib in macho_read_libraries(filename): if dylib not in MACHO_ALLOWED_LIBRARIES: print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) @@ -251,7 +227,7 @@ def pe_read_libraries(filename) -> List[str]: return libraries def check_PE_libraries(filename) -> bool: - ok = True + ok: bool = True for dylib in pe_read_libraries(filename): if dylib not in PE_ALLOWED_LIBRARIES: print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) @@ -284,7 +260,7 @@ def identify_executable(executable) -> Optional[str]: return None if __name__ == '__main__': - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) @@ -293,7 +269,7 @@ if __name__ == '__main__': retval = 1 continue - failed = [] + failed: List[str] = [] for (name, func) in CHECKS[etype]: if not func(filename): failed.append(name) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py new file mode 100755 index 0000000000..18ed7d61e0 --- /dev/null +++ b/contrib/devtools/test-symbol-check.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Test script for symbol-check.py +''' +import subprocess +import unittest + +def call_symbol_check(cc, source, executable, options): + subprocess.run([cc,source,'-o',executable] + options, check=True) + p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + return (p.returncode, p.stdout.rstrip()) + +def get_machine(cc): + p = subprocess.run([cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + return p.stdout.rstrip() + +class TestSymbolChecks(unittest.TestCase): + def test_ELF(self): + source = 'test1.c' + executable = 'test1' + cc = 'gcc' + + # there's no way to do this test for RISC-V at the moment; bionic's libc is 2.27 + # and we allow all symbols from 2.27. + if 'riscv' in get_machine(cc): + self.skipTest("test not available for RISC-V") + + # memfd_create was introduced in GLIBC 2.27, so is newer than the upper limit of + # all but RISC-V but still available on bionic + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #define _GNU_SOURCE + #include <sys/mman.h> + + int memfd_create(const char *name, unsigned int flags); + + int main() + { + memfd_create("test", 0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, []), + (1, executable + ': symbol memfd_create from unsupported version GLIBC_2.27\n' + + executable + ': failed IMPORTED_SYMBOLS')) + + # -lutil is part of the libc6 package so a safe bet that it's installed + # it's also out of context enough that it's unlikely to ever become a real dependency + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <utmp.h> + + int main() + { + login(0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), + (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + + executable + ': failed LIBRARY_DEPENDENCIES')) + + # finally, check a conforming file that simply uses a math function + source = 'test3.c' + executable = 'test3' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <math.h> + + int main() + { + return (int)pow(2.0, 4.0); + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (0, '')) + + def test_MACHO(self): + source = 'test1.c' + executable = 'test1' + cc = 'clang' + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <expat.h> + + int main() + { + XML_ExpatVersion(); + return 0; + } + + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat']), + (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed DYNAMIC_LIBRARIES')) + + source = 'test2.c' + executable = 'test2' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <CoreGraphics/CoreGraphics.h> + + int main() + { + CGMainDisplayID(); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics']), + (0, '')) + + def test_PE(self): + source = 'test1.c' + executable = 'test1.exe' + cc = 'x86_64-w64-mingw32-gcc' + + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <pdh.h> + + int main() + { + PdhConnectMachineA(NULL); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lpdh']), + (1, 'pdh.dll is not in ALLOWED_LIBRARIES!\n' + + executable + ': failed DYNAMIC_LIBRARIES')) + + source = 'test2.c' + executable = 'test2.exe' + with open(source, 'w', encoding="utf8") as f: + f.write(''' + #include <windows.h> + + int main() + { + CoFreeUnusedLibrariesEx(0,0); + return 0; + } + ''') + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lole32']), + (0, '')) + + +if __name__ == '__main__': + unittest.main() + diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index e005cb96da..db780ad53b 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index d498c9e2c8..f105968515 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index ec716cf2f3..0e2c5be0fa 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-linux-0.21" +name: "bitcoin-core-linux-22" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index a4f3219c22..2330ff7736 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -7,6 +7,7 @@ architectures: - "amd64" packages: - "faketime" +- "xorriso" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" @@ -18,7 +19,7 @@ script: | WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg genisoimage" + FAKETIME_PROGS="dmg xorrisofs" # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do @@ -36,5 +37,5 @@ script: | tar -xf ${UNSIGNED} OSX_VOLNAME="$(cat osx_volname)" ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app + ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index df50f45188..9a7dd13c9c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-osx-0.21" +name: "bitcoin-core-osx-22" enable_cache: true distro: "ubuntu" suites: @@ -28,6 +28,7 @@ packages: - "python3-dev" - "python3-setuptools" - "fonts-tuffy" +- "xorriso" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -37,10 +38,10 @@ script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin16" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" + HOSTS="x86_64-apple-darwin18" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" + FAKETIME_PROGS="ar ranlib date dmg xorrisofs" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 @@ -132,12 +133,11 @@ script: | make osx_volname make deploydir - OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} + cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} @@ -145,8 +145,7 @@ script: | find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd - make deploy - ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg + make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" cd installed find . -name "lib*.la" -delete diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 4d61cded74..fc79745e69 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-win-0.21" +name: "bitcoin-core-win-22" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 0a2c1302c8..7e28d27454 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -2,6 +2,8 @@ 617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach 152812300785C96444D3334D17565732E08E5E41 Andrew Chow +590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) +0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) 912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur @@ -22,12 +24,15 @@ B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke 07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford +AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) 9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo 77E72E69DA7EE0A148C06B21B34821D4944DE5F7 Nils Schneider +F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy 37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index dffcf99607..dbe1ea837b 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -40,25 +40,27 @@ Otherwise, follow the [Guix installation guide][guix/bin-install]. Guix allows us to achieve better binary security by using our CPU time to build everything from scratch. However, it doesn't sacrifice user choice in pursuit of -this: users can decide whether or not to bootstrap and to use substitutes. +this: users can decide whether or not to bootstrap and to use substitutes +(pre-built packages). After installation, you may want to consider [adding substitute -servers](#speeding-up-builds-with-substitute-servers) to speed up your build if -that fits your security model (say, if you're just testing that this works). -This is skippable if you're using the [Dockerfile][fanquake/guix-docker]. +servers](#speeding-up-builds-with-substitute-servers) from which to download +pre-built packages to speed up your build if that fits your security model (say, +if you're just testing that this works). Substitute servers are set up by +default if you're using the [Dockerfile][fanquake/guix-docker]. -If you prefer not to use any substitutes, make sure to set -`ADDITIONAL_GUIX_ENVIRONMENT_FLAGS` like the following snippet. The first build -will take a while, but the resulting packages will be cached for future builds. +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. ```sh -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--no-substitutes' +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ``` Likewise, to perform a bootstrapped build (takes even longer): ```sh -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap' ``` ### Using a version of Guix with `guix time-machine` capabilities @@ -82,17 +84,6 @@ export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ## Usage -### As a Development Environment - -For a Bitcoin Core depends development environment, simply invoke - -```sh -guix environment --manifest=contrib/guix/manifest.scm -``` - -And you'll land back in your shell with all the build dependencies required for -a `depends` build injected into your environment. - ### As a Tool for Deterministic Builds From the top of a clean Bitcoin Core repository: @@ -113,10 +104,8 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum * _**HOSTS**_ Override the space-separated list of platform triples for which to perform a - bootstrappable build. _(defaults to "x86\_64-linux-gnu - arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu")_ - - > Windows and OS X platform triplet support are WIP. + bootstrappable build. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf + aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32")_ * _**SOURCES_PATH**_ @@ -147,13 +136,29 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum string) is interpreted the same way as not setting `V` at all, and that `V=0` has the same effect as `V=1`. -* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ +* _**SUBSTITUTE_URLS**_ - Additional flags to be passed to `guix environment`. For a fully-bootstrapped + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#speeding-up-builds-with-substitute-servers) for more + details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. For a fully-bootstrapped build, set this to `--bootstrap --no-substitutes` (refer to the [security model section](#choosing-your-security-model) for more details). Note that a fully-bootstrapped build will take quite a long time on the first run. +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix environment` inside + `guix time-machine`. + ## Tips and Tricks ### Speeding up builds with substitute servers @@ -161,14 +166,15 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum _This whole section is automatically done in the convenience [Dockerfiles][fanquake/guix-docker]_ -For those who are used to life in the fast _(and trustful)_ lane, you can use -[substitute servers][guix/substitutes] to enable binary downloads of packages. +For those who are used to life in the fast _(and trustful)_ lane, you can +specify [substitute servers][guix/substitutes] from which to download pre-built +packages. > For those who only want to use substitutes from the official Guix build farm > and have authorized the build farm's signing key during Guix's installation, > you don't need to do anything. -#### Authorize the signing keys +#### Step 1: Authorize the signing keys For the official Guix build farm at https://ci.guix.gnu.org, run as root: @@ -182,7 +188,7 @@ For dongcarl's substitute server at https://guix.carldong.io, run as root: wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize ``` -#### Use the substitute servers +#### Step 2: Specify the substitute servers The official Guix build farm at https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag is supplied. @@ -196,7 +202,7 @@ To use dongcarl's substitute server for Bitcoin Core builds after having [authorized his signing key](#authorize-the-signing-keys): ``` -export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--substitute-urls="https://guix.carldong.io https://ci.guix.gnu.org"' +export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` ## FAQ @@ -212,9 +218,9 @@ As mentioned at the bottom of [this manual page][guix/bin-install]: ### When will Guix be packaged in debian? -Vagrant Cascadian has been making good progress on this -[here][debian/guix-package]. We have all the pieces needed to put up an APT -repository and will likely put one up soon. +Thanks to Vagrant Cascadian's diligent work, Guix is now [in debian +experimental][debian/guix-experimental]! Hopefully it will make its way into a +release soon. [b17e]: http://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ @@ -226,5 +232,5 @@ repository and will likely put one up soon. [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html -[debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 +[debian/guix-experimental]: https://packages.debian.org/experimental/guix [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 11d2c8b867..54cc5793f6 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -2,6 +2,116 @@ export LC_ALL=C set -e -o pipefail +################### +## Sanity Checks ## +################### + +################ +# Check 1: Make sure that we can invoke required tools +################ +for cmd in git make guix cat mkdir; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi +done + +################ +# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# Check 3: Make sure that we're not in a dirty worktree +################ +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +else + GIT_COMMIT=$(git rev-parse --short=12 HEAD) +fi + +################ +# Check 4: Make sure that build directories do no exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu + x86_64-w64-mingw32}" + +DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +######### +# Setup # +######### + # Determine the maximum number of jobs to run simultaneously (overridable by # environment) MAX_JOBS="${MAX_JOBS:-$(nproc)}" @@ -16,11 +126,23 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. time-machine() { + # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ + --max-jobs="$MAX_JOBS" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" } +# Make sure an output directory exists for our builds +OUTDIR="${OUTDIR:-${PWD}/output}" +[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" + +######### +# Build # +######### + # Function to be called when building for host ${1} and the user interrupts the # build int_trap() { @@ -38,9 +160,9 @@ and untracked files and directories will be wiped, allowing you to start anew. EOF } -# Deterministically build Bitcoin Core for HOSTs (overridable by environment) +# Deterministically build Bitcoin Core # shellcheck disable=SC2153 -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do +for host in $HOSTS; do # Display proper warning when the user interrupts the build trap 'int_trap ${host}' INT @@ -50,6 +172,19 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # for the particular $HOST we're building for export HOST="$host" + # shellcheck disable=SC2030 +cat << EOF +INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: + ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} + ...running at most ${MAX_JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/bitcoin' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '${OUTDIR:?not set}' + ...bind-mounted in container to: '/outdir' +EOF + # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm'. # @@ -99,20 +234,36 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv # make the downloaded depends sources available to it. The sources # should have been downloaded prior to this invocation. # - # shellcheck disable=SC2086 + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ --container \ --pure \ --no-cwd \ --share="$PWD"=/bitcoin \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR"=/outdir \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + --max-jobs="$MAX_JOBS" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ MAX_JOBS="$MAX_JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ + OUTDIR=/outdir \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index d658c4f6a6..b00c42ce01 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -11,9 +11,15 @@ if [ -n "$V" ]; then export VERBOSE="$V" fi -# Check that environment variables assumed to be set by the environment are set -echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." -echo "At most ${MAX_JOBS:?not set} jobs will run at once..." +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + MAX_JOBS: ${MAX_JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF ##################### # Environment Setup # @@ -23,19 +29,6 @@ echo "At most ${MAX_JOBS:?not set} jobs will run at once..." # $HOSTs after successfully building. BASEPREFIX="${PWD}/depends" -# Setup an output directory for our build -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" - -# Setup the directory where our Bitcoin Core build for HOST will occur -DISTSRC="${DISTSRC:-${PWD}/distsrc-${HOST}}" -if [ -e "$DISTSRC" ]; then - echo "DISTSRC directory '${DISTSRC}' exists, probably because of previous builds... Aborting..." - exit 1 -else - mkdir -p "$DISTSRC" -fi - # Given a package name and an output name, return the path of that output in our # current guix environment store_path() { @@ -189,6 +182,7 @@ esac # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +mkdir -p "$DISTSRC" ( cd "$DISTSRC" diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index fe677e3a1f..2d9a4a2153 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -6,11 +6,7 @@ The `macdeployqtplus` script should not be run manually. Instead, after building make deploy ``` -During the deployment process, the disk image window will pop up briefly -when the fancy settings are applied. This is normal, please do not interfere, -the process will unmount the DMG and cleanup before finishing. - -When complete, it will have produced `Bitcoin-Qt.dmg`. +When complete, it will have produced `Bitcoin-Core.dmg`. ## SDK Extraction @@ -96,22 +92,18 @@ created using these tools. The build process has been designed to avoid includin SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely redistributed. -`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been -patched. A system `genisoimage` will work fine, but it will not be deterministic because -the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch). -No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for -a single invocation, so that's no real concern. +[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. -`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its -maintainer has seemingly abandoned the project. +`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this +tool and its maintainer has seemingly abandoned the project. The DMG tool has the ability to create DMGs from scratch as well, but this functionality is broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `genisoimage` would no longer be necessary. +and `xorrisofs` would no longer be necessary. Background images and other features can be added to DMG files by inserting a -`.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`. +`.DS_Store` during creation. As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be diff --git a/contrib/macdeploy/custom_dsstore.py b/contrib/macdeploy/custom_dsstore.py deleted file mode 100755 index dc1c1882dd..0000000000 --- a/contrib/macdeploy/custom_dsstore.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2013-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -import biplist -from ds_store import DSStore -from mac_alias import Alias -import sys - -output_file = sys.argv[1] -package_name_ns = sys.argv[2] - -ds = DSStore.open(output_file, 'w+') -ds['.']['bwsp'] = { - 'ShowStatusBar': False, - 'WindowBounds': '{{300, 280}, {500, 343}}', - 'ContainerShowSidebar': False, - 'SidebarWidth': 0, - 'ShowTabView': False, - 'PreviewPaneVisibility': False, - 'ShowToolbar': False, - 'ShowSidebar': False, - 'ShowPathbar': True -} - -icvp = { - 'gridOffsetX': 0.0, - 'textSize': 12.0, - 'viewOptionsVersion': 1, - 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', - 'backgroundColorBlue': 1.0, - 'iconSize': 96.0, - 'backgroundColorGreen': 1.0, - 'arrangeBy': 'none', - 'showIconPreview': True, - 'gridSpacing': 100.0, - 'gridOffsetY': 0.0, - 'showItemInfo': False, - 'labelOnBottom': True, - 'backgroundType': 2, - 'backgroundColorRed': 1.0 -} -alias = Alias.from_bytes(icvp['backgroundImageAlias']) -alias.volume.name = package_name_ns -alias.volume.posix_path = '/Volumes/' + package_name_ns -alias.volume.disk_image_alias.target.filename = package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.carbon_path = 'Macintosh HD:Users:\x00bitcoinuser:\x00Documents:\x00bitcoin:\x00bitcoin:\x00' + package_name_ns + '.temp.dmg' -alias.volume.disk_image_alias.target.posix_path = 'Users/bitcoinuser/Documents/bitcoin/bitcoin/' + package_name_ns + '.temp.dmg' -alias.target.carbon_path = package_name_ns + ':.background:\x00background.tiff' -icvp['backgroundImageAlias'] = biplist.Data(alias.to_bytes()) -ds['.']['icvp'] = icvp - -ds['.']['vSrn'] = ('long', 1) - -ds['Applications']['Iloc'] = (370, 156) -ds['Bitcoin-Qt.app']['Iloc'] = (128, 156) - -ds.flush() -ds.close() diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh deleted file mode 100755 index 3c7bdf4217..0000000000 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -INPUTFILE="Xcode_7.3.1.dmg" -HFSFILENAME="5.hfs" -SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk" - -7z x "${INPUTFILE}" "${HFSFILENAME}" -SDKNAME="$(basename "${SDKDIR}")" -SDKDIRINODE=$(ifind -n "${SDKDIR}" "${HFSFILENAME}") -fls "${HFSFILENAME}" -rpF ${SDKDIRINODE} | - while read type inode filename; do - inode="${inode::-1}" - if [ "${filename:0:14}" = "usr/share/man/" ]; then - continue - fi - filename="${SDKNAME}/$filename" - echo "Extracting $filename ..." - mkdir -p "$(dirname "$filename")" - if [ "$type" = "l/l" ]; then - ln -s "$(icat "${HFSFILENAME}" $inode)" "$filename" - else - icat "${HFSFILENAME}" $inode >"$filename" - fi -done -echo "Building ${SDKNAME}.tar.gz ..." -MTIME="$(istat "${HFSFILENAME}" "${SDKDIRINODE}" | perl -nle 'm/Content Modified:\s+(.*?)\s\(/ && print $1')" -find "${SDKNAME}" | sort | tar --no-recursion --mtime="${MTIME}" --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > "${SDKNAME}.tar.gz" -echo 'All done!' diff --git a/contrib/macdeploy/fancy.plist b/contrib/macdeploy/fancy.plist deleted file mode 100644 index ef277a7f14..0000000000 --- a/contrib/macdeploy/fancy.plist +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>window_bounds</key> - <array> - <integer>300</integer> - <integer>300</integer> - <integer>800</integer> - <integer>620</integer> - </array> - <key>background_picture</key> - <string>background.tiff</string> - <key>icon_size</key> - <integer>96</integer> - <key>applications_symlink</key> - <true/> - <key>items_position</key> - <dict> - <key>Applications</key> - <array> - <integer>370</integer> - <integer>156</integer> - </array> - <key>Bitcoin-Qt.app</key> - <array> - <integer>128</integer> - <integer>156</integer> - </array> - </dict> -</dict> -</plist> diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 524104398b..9bf3305288 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -16,9 +16,13 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # -import subprocess, sys, re, os, shutil, stat, os.path, time -from string import Template +import plistlib +import sys, re, os, shutil, stat, os.path from argparse import ArgumentParser +from ds_store import DSStore +from mac_alias import Alias +from pathlib import Path +from subprocess import PIPE, run from typing import List, Optional # This is ported from the original macdeployqt with modifications @@ -49,28 +53,18 @@ class FrameworkInfo(object): return False def __str__(self): - return """ Framework name: {} - Framework directory: {} - Framework path: {} - Binary name: {} - Binary directory: {} - Binary path: {} - Version: {} - Install name: {} - Deployed install name: {} - Source file Path: {} - Deployed Directory (relative to bundle): {} -""".format(self.frameworkName, - self.frameworkDirectory, - self.frameworkPath, - self.binaryName, - self.binaryDirectory, - self.binaryPath, - self.version, - self.installName, - self.deployedInstallName, - self.sourceFilePath, - self.destinationDirectory) + return f""" Framework name: {frameworkName} + Framework directory: {self.frameworkDirectory} + Framework path: {self.frameworkPath} + Binary name: {self.binaryName} + Binary directory: {self.binaryDirectory} + Binary path: {self.binaryPath} + Version: {self.version} + Install name: {self.installName} + Deployed install name: {self.deployedInstallName} + Source file Path: {self.sourceFilePath} + Deployed Directory (relative to bundle): {self.destinationDirectory} +""" def isDylib(self): return self.frameworkName.endswith(".dylib") @@ -97,7 +91,7 @@ class FrameworkInfo(object): m = cls.reOLine.match(line) if m is None: - raise RuntimeError("otool line could not be parsed: " + line) + raise RuntimeError(f"otool line could not be parsed: {line}") path = m.group(1) @@ -117,7 +111,7 @@ class FrameworkInfo(object): info.version = "-" info.installName = path - info.deployedInstallName = "@executable_path/../Frameworks/" + info.binaryName + info.deployedInstallName = f"@executable_path/../Frameworks/{info.binaryName}" info.sourceFilePath = path info.destinationDirectory = cls.bundleFrameworkDirectory else: @@ -129,7 +123,7 @@ class FrameworkInfo(object): break i += 1 if i == len(parts): - raise RuntimeError("Could not find .framework or .dylib in otool line: " + line) + raise RuntimeError(f"Could not find .framework or .dylib in otool line: {line}") info.frameworkName = parts[i] info.frameworkDirectory = "/".join(parts[:i]) @@ -140,7 +134,7 @@ class FrameworkInfo(object): info.binaryPath = os.path.join(info.binaryDirectory, info.binaryName) info.version = parts[i+2] - info.deployedInstallName = "@executable_path/../Frameworks/" + os.path.join(info.frameworkName, info.binaryPath) + info.deployedInstallName = f"@executable_path/../Frameworks/{os.path.join(info.frameworkName, info.binaryPath)}" info.destinationDirectory = os.path.join(cls.bundleFrameworkDirectory, info.frameworkName, info.binaryDirectory) info.sourceResourcesDirectory = os.path.join(info.frameworkPath, "Resources") @@ -154,10 +148,10 @@ class FrameworkInfo(object): class ApplicationBundleInfo(object): def __init__(self, path: str): self.path = path - appName = "Bitcoin-Qt" - self.binaryPath = os.path.join(path, "Contents", "MacOS", appName) + # for backwards compatibility reasons, this must remain as Bitcoin-Qt + self.binaryPath = os.path.join(path, "Contents", "MacOS", "Bitcoin-Qt") if not os.path.exists(self.binaryPath): - raise RuntimeError("Could not find bundle binary for " + path) + raise RuntimeError(f"Could not find bundle binary for {path}") self.resourcesPath = os.path.join(path, "Contents", "Resources") self.pluginPath = os.path.join(path, "Contents", "PlugIns") @@ -181,30 +175,26 @@ class DeploymentInfo(object): self.pluginPath = pluginPath def usesFramework(self, name: str) -> bool: - nameDot = "{}.".format(name) - libNameDot = "lib{}.".format(name) for framework in self.deployedFrameworks: if framework.endswith(".framework"): - if framework.startswith(nameDot): + if framework.startswith(f"{name}."): return True elif framework.endswith(".dylib"): - if framework.startswith(libNameDot): + if framework.startswith(f"lib{name}."): return True return False def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: - if verbose >= 3: - print("Inspecting with otool: " + binaryPath) + if verbose: + print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = subprocess.Popen([otoolbin, "-L", binaryPath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - o_stdout, o_stderr = otool.communicate() + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) if otool.returncode != 0: - if verbose >= 1: - sys.stderr.write(o_stderr) - sys.stderr.flush() - raise RuntimeError("otool failed with return code {}".format(otool.returncode)) + sys.stderr.write(otool.stderr) + sys.stderr.flush() + raise RuntimeError(f"otool failed with return code {otool.returncode}") - otoolLines = o_stdout.split("\n") + otoolLines = otool.stdout.split("\n") otoolLines.pop(0) # First line is the inspected binary if ".framework" in binaryPath or binaryPath.endswith(".dylib"): otoolLines.pop(0) # Frameworks and dylibs list themselves as a dependency. @@ -214,7 +204,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: line = line.replace("@loader_path", os.path.dirname(binaryPath)) info = FrameworkInfo.fromOtoolLibraryLine(line.strip()) if info is not None: - if verbose >= 3: + if verbose: print("Found framework:") print(info) libraries.append(info) @@ -223,10 +213,10 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: def runInstallNameTool(action: str, *args): installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") - subprocess.check_call([installnametoolbin, "-"+action] + list(args)) + run([installnametoolbin, "-"+action] + list(args), check=True) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): - if verbose >= 3: + if verbose: print("Using install_name_tool:") print(" in", binaryPath) print(" change reference", oldName) @@ -234,7 +224,7 @@ def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int) runInstallNameTool("change", oldName, newName, binaryPath) def changeIdentification(id: str, binaryPath: str, verbose: int): - if verbose >= 3: + if verbose: print("Using install_name_tool:") print(" change identification in", binaryPath) print(" to", id) @@ -242,22 +232,22 @@ def changeIdentification(id: str, binaryPath: str, verbose: int): def runStrip(binaryPath: str, verbose: int): stripbin=os.getenv("STRIP", "strip") - if verbose >= 3: + if verbose: print("Using strip:") print(" stripped", binaryPath) - subprocess.check_call([stripbin, "-x", binaryPath]) + run([stripbin, "-x", binaryPath], check=True) def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional[str]: if framework.sourceFilePath.startswith("Qt"): #standard place for Nokia Qt installer's frameworks - fromPath = "/Library/Frameworks/" + framework.sourceFilePath + fromPath = f"/Library/Frameworks/{framework.sourceFilePath}" else: fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) if not os.path.exists(fromPath): - raise RuntimeError("No file at " + fromPath) + raise RuntimeError(f"No file at {fromPath}") if os.path.exists(toPath): return None # Already there @@ -266,7 +256,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional os.makedirs(toDir) shutil.copy2(fromPath, toPath) - if verbose >= 3: + if verbose: print("Copied:", fromPath) print(" to:", toPath) @@ -280,13 +270,12 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional linkto = framework.version if not os.path.exists(linkfrom): os.symlink(linkto, linkfrom) - if verbose >= 2: - print("Linked:", linkfrom, "->", linkto) + print("Linked:", linkfrom, "->", linkto) fromResourcesDir = framework.sourceResourcesDirectory if os.path.exists(fromResourcesDir): toResourcesDir = os.path.join(path, framework.destinationResourcesDirectory) shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) - if verbose >= 3: + if verbose: print("Copied resources:", fromResourcesDir) print(" to:", toResourcesDir) fromContentsDir = framework.sourceVersionContentsDirectory @@ -295,7 +284,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - if verbose >= 3: + if verbose: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) elif framework.frameworkName.startswith("libQtGui"): # Copy qt_menu.nib (applies to non-framework layout) @@ -303,7 +292,7 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional qtMenuNibDestinationPath = os.path.join(path, "Contents", "Resources", "qt_menu.nib") if os.path.exists(qtMenuNibSourcePath) and not os.path.exists(qtMenuNibDestinationPath): shutil.copytree(qtMenuNibSourcePath, qtMenuNibDestinationPath, symlinks=True) - if verbose >= 3: + if verbose: print("Copied for libQtGui:", qtMenuNibSourcePath) print(" to:", qtMenuNibDestinationPath) @@ -317,16 +306,14 @@ def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPat framework = frameworks.pop(0) deploymentInfo.deployedFrameworks.append(framework.frameworkName) - if verbose >= 2: - print("Processing", framework.frameworkName, "...") + print("Processing", framework.frameworkName, "...") # Get the Qt path from one of the Qt frameworks if deploymentInfo.qtPath is None and framework.isQtFramework(): deploymentInfo.detectQtPath(framework.frameworkDirectory) if framework.installName.startswith("@executable_path") or framework.installName.startswith(bundlePath): - if verbose >= 2: - print(framework.frameworkName, "already deployed, skipping.") + print(framework.frameworkName, "already deployed, skipping.") continue # install_name_tool the new id into the binary @@ -357,8 +344,8 @@ def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPat def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip: bool, verbose: int) -> DeploymentInfo: frameworks = getFrameworks(applicationBundle.binaryPath, verbose) - if len(frameworks) == 0 and verbose >= 1: - print("Warning: Could not find any external frameworks to deploy in {}.".format(applicationBundle.path)) + if len(frameworks) == 0: + print(f"Warning: Could not find any external frameworks to deploy in {applicationBundle.path}.") return DeploymentInfo() else: return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) @@ -477,8 +464,7 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme plugins.append((pluginDirectory, pluginName)) for pluginDirectory, pluginName in plugins: - if verbose >= 2: - print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") + print("Processing plugin", os.path.join(pluginDirectory, pluginName), "...") sourcePath = os.path.join(deploymentInfo.pluginPath, pluginDirectory, pluginName) destinationDirectory = os.path.join(appBundleInfo.pluginPath, pluginDirectory) @@ -487,7 +473,7 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme destinationPath = os.path.join(destinationDirectory, pluginName) shutil.copy2(sourcePath, destinationPath) - if verbose >= 3: + if verbose: print("Copied:", sourcePath) print(" to:", destinationPath) @@ -503,147 +489,50 @@ def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: Deployme if dependency.frameworkName not in deploymentInfo.deployedFrameworks: deployFrameworks([dependency], appBundleInfo.path, destinationPath, strip, verbose, deploymentInfo) -qt_conf="""[Paths] -Translations=Resources -Plugins=PlugIns -""" - ap = ArgumentParser(description="""Improved version of macdeployqt. Outputs a ready-to-deploy app in a folder "dist" and optionally wraps it in a .dmg file. Note, that the "dist" folder will be deleted before deploying on each run. -Optionally, Qt translation files (.qm) and additional resources can be added to the bundle. - -Also optionally signs the .app bundle; set the CODESIGNARGS environment variable to pass arguments -to the codesign tool. -E.g. CODESIGNARGS='--sign "Developer ID Application: ..." --keychain /encrypted/foo.keychain'""") +Optionally, Qt translation files (.qm) can be added to the bundle.""") ap.add_argument("app_bundle", nargs=1, metavar="app-bundle", help="application bundle to be deployed") -ap.add_argument("-verbose", type=int, nargs=1, default=[1], metavar="<0-3>", help="0 = no output, 1 = error/warning (default), 2 = normal, 3 = debug") +ap.add_argument("appname", nargs=1, metavar="appname", help="name of the app being deployed") +ap.add_argument("-verbose", nargs="?", const=True, help="Output additional debugging information") ap.add_argument("-no-plugins", dest="plugins", action="store_false", default=True, help="skip plugin deployment") ap.add_argument("-no-strip", dest="strip", action="store_false", default=True, help="don't run 'strip' on the binaries") -ap.add_argument("-sign", dest="sign", action="store_true", default=False, help="sign .app bundle with codesign tool") -ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image; if basename is not specified, a camel-cased version of the app name is used") -ap.add_argument("-fancy", nargs=1, metavar="plist", default=[], help="make a fancy looking disk image using the given plist file with instructions; requires -dmg to work") -ap.add_argument("-add-qt-tr", nargs=1, metavar="languages", default=[], help="add Qt translation files to the bundle's resources; the language list must be separated with commas, not with whitespace") -ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translation files") -ap.add_argument("-add-resources", nargs="+", metavar="path", default=[], help="list of additional files or folders to be copied into the bundle's resources; must be the last argument") -ap.add_argument("-volname", nargs=1, metavar="volname", default=[], help="custom volume name for dmg") +ap.add_argument("-dmg", nargs="?", const="", metavar="basename", help="create a .dmg disk image") +ap.add_argument("-translations-dir", nargs=1, metavar="path", default=None, help="Path to Qt's translations. Base translations will automatically be added to the bundle's resources.") config = ap.parse_args() -verbose = config.verbose[0] +verbose = config.verbose # ------------------------------------------------ app_bundle = config.app_bundle[0] +appname = config.appname[0] if not os.path.exists(app_bundle): - if verbose >= 1: - sys.stderr.write("Error: Could not find app bundle \"{}\"\n".format(app_bundle)) + sys.stderr.write(f"Error: Could not find app bundle \"{app_bundle}\"\n") sys.exit(1) -app_bundle_name = os.path.splitext(os.path.basename(app_bundle))[0] - -# ------------------------------------------------ -translations_dir = None -if config.translations_dir and config.translations_dir[0]: - if os.path.exists(config.translations_dir[0]): - translations_dir = config.translations_dir[0] - else: - if verbose >= 1: - sys.stderr.write("Error: Could not find translation dir \"{}\"\n".format(translations_dir)) - sys.exit(1) -# ------------------------------------------------ - -for p in config.add_resources: - if verbose >= 3: - print("Checking for \"%s\"..." % p) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find additional resource file \"{}\"\n".format(p)) - sys.exit(1) - -# ------------------------------------------------ - -if len(config.fancy) == 1: - if verbose >= 3: - print("Fancy: Importing plistlib...") - try: - import plistlib - except ImportError: - if verbose >= 1: - sys.stderr.write("Error: Could not import plistlib which is required for fancy disk images.\n") - sys.exit(1) - - p = config.fancy[0] - if verbose >= 3: - print("Fancy: Loading \"{}\"...".format(p)) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - try: - with open(p, 'rb') as fp: - fancy = plistlib.load(fp, fmt=plistlib.FMT_XML) - except: - if verbose >= 1: - sys.stderr.write("Error: Could not parse fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - try: - assert "window_bounds" not in fancy or (isinstance(fancy["window_bounds"], list) and len(fancy["window_bounds"]) == 4) - assert "background_picture" not in fancy or isinstance(fancy["background_picture"], str) - assert "icon_size" not in fancy or isinstance(fancy["icon_size"], int) - assert "applications_symlink" not in fancy or isinstance(fancy["applications_symlink"], bool) - if "items_position" in fancy: - assert isinstance(fancy["items_position"], dict) - for key, value in fancy["items_position"].items(): - assert isinstance(value, list) and len(value) == 2 and isinstance(value[0], int) and isinstance(value[1], int) - except: - if verbose >= 1: - sys.stderr.write("Error: Bad format of fancy disk image plist at \"{}\"\n".format(p)) - sys.exit(1) - - if "background_picture" in fancy: - bp = fancy["background_picture"] - if verbose >= 3: - print("Fancy: Resolving background picture \"{}\"...".format(bp)) - if not os.path.exists(bp): - bp = os.path.join(os.path.dirname(p), bp) - if not os.path.exists(bp): - if verbose >= 1: - sys.stderr.write("Error: Could not find background picture at \"{}\" or \"{}\"\n".format(fancy["background_picture"], bp)) - sys.exit(1) - else: - fancy["background_picture"] = bp -else: - fancy = None - # ------------------------------------------------ if os.path.exists("dist"): - if verbose >= 2: - print("+ Removing old dist folder +") - + print("+ Removing existing dist folder +") shutil.rmtree("dist") -# ------------------------------------------------ - -if len(config.volname) == 1: - volname = config.volname[0] -else: - volname = app_bundle_name +if os.path.exists(appname + ".dmg"): + print("+ Removing existing DMG +") + os.unlink(appname + ".dmg") # ------------------------------------------------ target = os.path.join("dist", "Bitcoin-Qt.app") -if verbose >= 2: - print("+ Copying source bundle +") -if verbose >= 3: +print("+ Copying source bundle +") +if verbose: print(app_bundle, "->", target) os.mkdir("dist") @@ -653,257 +542,154 @@ applicationBundle = ApplicationBundleInfo(target) # ------------------------------------------------ -if verbose >= 2: - print("+ Deploying frameworks +") +print("+ Deploying frameworks +") try: deploymentInfo = deployFrameworksForAppBundle(applicationBundle, config.strip, verbose) if deploymentInfo.qtPath is None: deploymentInfo.qtPath = os.getenv("QTDIR", None) if deploymentInfo.qtPath is None: - if verbose >= 1: - sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") + sys.stderr.write("Warning: Could not detect Qt's path, skipping plugin deployment!\n") config.plugins = False except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: {}\n".format(str(e))) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ if config.plugins: - if verbose >= 2: - print("+ Deploying plugins +") + print("+ Deploying plugins +") try: deployPlugins(applicationBundle, deploymentInfo, config.strip, verbose) except RuntimeError as e: - if verbose >= 1: - sys.stderr.write("Error: {}\n".format(str(e))) + sys.stderr.write(f"Error: {str(e)}\n") sys.exit(1) # ------------------------------------------------ -if len(config.add_qt_tr) == 0: - add_qt_tr = [] -else: - if translations_dir is not None: - qt_tr_dir = translations_dir - else: - if deploymentInfo.qtPath is not None: - qt_tr_dir = os.path.join(deploymentInfo.qtPath, "translations") - else: - sys.stderr.write("Error: Could not find Qt translation path\n") - sys.exit(1) - add_qt_tr = ["qt_{}.qm".format(lng) for lng in config.add_qt_tr[0].split(",")] - for lng_file in add_qt_tr: - p = os.path.join(qt_tr_dir, lng_file) - if verbose >= 3: - print("Checking for \"{}\"...".format(p)) - if not os.path.exists(p): - if verbose >= 1: - sys.stderr.write("Error: Could not find Qt translation file \"{}\"\n".format(lng_file)) - sys.exit(1) +if config.translations_dir: + if not Path(config.translations_dir[0]).exists(): + sys.stderr.write(f"Error: Could not find translation dir \"{config.translations_dir[0]}\"\n") + sys.exit(1) + +print("+ Adding Qt translations +") + +translations = Path(config.translations_dir[0]) + +regex = re.compile('qt_[a-z]*(.qm|_[A-Z]*.qm)') + +lang_files = [x for x in translations.iterdir() if regex.match(x.name)] + +for file in lang_files: + if verbose: + print(file.as_posix(), "->", os.path.join(applicationBundle.resourcesPath, file.name)) + shutil.copy2(file.as_posix(), os.path.join(applicationBundle.resourcesPath, file.name)) # ------------------------------------------------ -if verbose >= 2: - print("+ Installing qt.conf +") +print("+ Installing qt.conf +") + +qt_conf="""[Paths] +Translations=Resources +Plugins=PlugIns +""" with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: f.write(qt_conf.encode()) # ------------------------------------------------ -if len(add_qt_tr) > 0 and verbose >= 2: - print("+ Adding Qt translations +") - -for lng_file in add_qt_tr: - if verbose >= 3: - print(os.path.join(qt_tr_dir, lng_file), "->", os.path.join(applicationBundle.resourcesPath, lng_file)) - shutil.copy2(os.path.join(qt_tr_dir, lng_file), os.path.join(applicationBundle.resourcesPath, lng_file)) +print("+ Generating .DS_Store +") + +output_file = os.path.join("dist", ".DS_Store") + +ds = DSStore.open(output_file, 'w+') + +ds['.']['bwsp'] = { + 'WindowBounds': '{{300, 280}, {500, 343}}', + 'PreviewPaneVisibility': False, +} + +icvp = { + 'gridOffsetX': 0.0, + 'textSize': 12.0, + 'viewOptionsVersion': 1, + 'backgroundImageAlias': b'\x00\x00\x00\x00\x02\x1e\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd1\x94\\\xb0H+\x00\x05\x00\x00\x00\x98\x0fbackground.tiff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x99\xd19\xb0\xf8\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\r\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b.background\x00\x00\x10\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x11\x00\x08\x00\x00\xd19\xb0\xf8\x00\x00\x00\x01\x00\x04\x00\x00\x00\x98\x00\x0e\x00 \x00\x0f\x00b\x00a\x00c\x00k\x00g\x00r\x00o\x00u\x00n\x00d\x00.\x00t\x00i\x00f\x00f\x00\x0f\x00\x02\x00\x00\x00\x12\x00\x1c/.background/background.tiff\x00\x14\x01\x06\x00\x00\x00\x00\x01\x06\x00\x02\x00\x00\x0cMacintosh HD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xce\x97\xab\xc3H+\x00\x00\x01\x88[\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02u\xab\x8d\xd1\x94\\\xb0devrddsk\xff\xff\xff\xff\x00\x00\t \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x07bitcoin\x00\x00\x10\x00\x08\x00\x00\xce\x97\xab\xc3\x00\x00\x00\x11\x00\x08\x00\x00\xd1\x94\\\xb0\x00\x00\x00\x01\x00\x14\x01\x88[\x88\x00\x16\xa9\t\x00\x08\xfaR\x00\x08\xfaQ\x00\x02d\x8e\x00\x0e\x00\x02\x00\x00\x00\x0f\x00\x1a\x00\x0c\x00M\x00a\x00c\x00i\x00n\x00t\x00o\x00s\x00h\x00 \x00H\x00D\x00\x13\x00\x01/\x00\x00\x15\x00\x02\x00\x14\xff\xff\x00\x00\xff\xff\x00\x00', + 'backgroundColorBlue': 1.0, + 'iconSize': 96.0, + 'backgroundColorGreen': 1.0, + 'arrangeBy': 'none', + 'showIconPreview': True, + 'gridSpacing': 100.0, + 'gridOffsetY': 0.0, + 'showItemInfo': False, + 'labelOnBottom': True, + 'backgroundType': 2, + 'backgroundColorRed': 1.0 +} +alias = Alias().from_bytes(icvp['backgroundImageAlias']) +alias.volume.name = appname +alias.volume.posix_path = '/Volumes/' + appname +icvp['backgroundImageAlias'] = alias.to_bytes() +ds['.']['icvp'] = icvp + +ds['.']['vSrn'] = ('long', 1) + +ds['Applications']['Iloc'] = (370, 156) +ds['Bitcoin-Qt.app']['Iloc'] = (128, 156) + +ds.flush() +ds.close() # ------------------------------------------------ -if len(config.add_resources) > 0 and verbose >= 2: - print("+ Adding additional resources +") +if config.dmg is not None: -for p in config.add_resources: - t = os.path.join(applicationBundle.resourcesPath, os.path.basename(p)) - if verbose >= 3: - print(p, "->", t) - if os.path.isdir(p): - shutil.copytree(p, t, symlinks=True) - else: - shutil.copy2(p, t) + print("+ Preparing .dmg disk image +") -# ------------------------------------------------ + if verbose: + print("Determining size of \"dist\"...") + size = 0 + for path, dirs, files in os.walk("dist"): + for file in files: + size += os.path.getsize(os.path.join(path, file)) + size += int(size * 0.15) -if config.sign and 'CODESIGNARGS' not in os.environ: - print("You must set the CODESIGNARGS environment variable. Skipping signing.") -elif config.sign: - if verbose >= 1: - print("Code-signing app bundle {}".format(target)) - subprocess.check_call("codesign --force {} {}".format(os.environ['CODESIGNARGS'], target), shell=True) + if verbose: + print("Creating temp image for modification...") -# ------------------------------------------------ + tempname: str = appname + ".temp.dmg" -if config.dmg is not None: + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) - def runHDIUtil(verb: str, image_basename: str, **kwargs) -> int: - hdiutil_args = ["hdiutil", verb, image_basename + ".dmg"] - if "capture_stdout" in kwargs: - del kwargs["capture_stdout"] - run = subprocess.check_output - else: - if verbose < 2: - hdiutil_args.append("-quiet") - elif verbose >= 3: - hdiutil_args.append("-verbose") - run = subprocess.check_call - - for key, value in kwargs.items(): - hdiutil_args.append("-" + key) - if value is not True: - hdiutil_args.append(str(value)) - - return run(hdiutil_args, universal_newlines=True) - - if verbose >= 2: - if fancy is None: - print("+ Creating .dmg disk image +") - else: - print("+ Preparing .dmg disk image +") - - if config.dmg != "": - dmg_name = config.dmg - else: - spl = app_bundle_name.split(" ") - dmg_name = spl[0] + "".join(p.capitalize() for p in spl[1:]) - - if fancy is None: - try: - runHDIUtil("create", dmg_name, srcfolder="dist", format="UDBZ", volname=volname, ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - else: - if verbose >= 3: - print("Determining size of \"dist\"...") - size = 0 - for path, dirs, files in os.walk("dist"): - for file in files: - size += os.path.getsize(os.path.join(path, file)) - size += int(size * 0.15) - - if verbose >= 3: - print("Creating temp image for modification...") - try: - runHDIUtil("create", dmg_name + ".temp", srcfolder="dist", format="UDRW", size=size, volname=volname, ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - if verbose >= 3: - print("Attaching temp image...") - try: - output = runHDIUtil("attach", dmg_name + ".temp", readwrite=True, noverify=True, noautoopen=True, capture_stdout=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - disk_name = m.group(1) - - if verbose >= 2: - print("+ Applying fancy settings +") - - if "background_picture" in fancy: - bg_path = os.path.join(disk_root, ".background", os.path.basename(fancy["background_picture"])) - os.mkdir(os.path.dirname(bg_path)) - if verbose >= 3: - print(fancy["background_picture"], "->", bg_path) - shutil.copy2(fancy["background_picture"], bg_path) - else: - bg_path = None - - if fancy.get("applications_symlink", False): - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - - # The Python appscript package broke with OSX 10.8 and isn't being fixed. - # So we now build up an AppleScript string and use the osascript command - # to make the .dmg file pretty: - appscript = Template( """ - on run argv - tell application "Finder" - tell disk "$disk" - open - set current view of container window to icon view - set toolbar visible of container window to false - set statusbar visible of container window to false - set the bounds of container window to {$window_bounds} - set theViewOptions to the icon view options of container window - set arrangement of theViewOptions to not arranged - set icon size of theViewOptions to $icon_size - $background_commands - $items_positions - close -- close/reopen works around a bug... - open - update without registering applications - delay 5 - eject - end tell - end tell - end run - """) - - itemscript = Template('set position of item "${item}" of container window to {${position}}') - items_positions = [] - if "items_position" in fancy: - for name, position in fancy["items_position"].items(): - params = { "item" : name, "position" : ",".join([str(p) for p in position]) } - items_positions.append(itemscript.substitute(params)) - - params = { - "disk" : volname, - "window_bounds" : "300,300,800,620", - "icon_size" : "96", - "background_commands" : "", - "items_positions" : "\n ".join(items_positions) - } - if "window_bounds" in fancy: - params["window_bounds"] = ",".join([str(p) for p in fancy["window_bounds"]]) - if "icon_size" in fancy: - params["icon_size"] = str(fancy["icon_size"]) - if bg_path is not None: - # Set background file, then call SetFile to make it invisible. - # (note: making it invisible first makes set background picture fail) - bgscript = Template("""set background picture of theViewOptions to file ".background:$bgpic" - do shell script "SetFile -a V /Volumes/$disk/.background/$bgpic" """) - params["background_commands"] = bgscript.substitute({"bgpic" : os.path.basename(bg_path), "disk" : params["disk"]}) - - s = appscript.substitute(params) - if verbose >= 2: - print("Running AppleScript:") - print(s) - - p = subprocess.Popen(['osascript', '-'], stdin=subprocess.PIPE) - p.communicate(input=s.encode('utf-8')) - if p.returncode: - print("Error running osascript.") - - if verbose >= 2: - print("+ Finalizing .dmg disk image +") - time.sleep(5) - - try: - runHDIUtil("convert", dmg_name + ".temp", format="UDBZ", o=dmg_name + ".dmg", ov=True) - except subprocess.CalledProcessError as e: - sys.exit(e.returncode) - - os.unlink(dmg_name + ".temp.dmg") + if verbose: + print("Attaching temp image...") + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout + + m = re.search(r"/Volumes/(.+$)", output) + disk_root = m.group(0) + + print("+ Applying fancy settings +") + + bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) + os.mkdir(os.path.dirname(bg_path)) + if verbose: + print('background.tiff', "->", bg_path) + shutil.copy2('background.tiff', bg_path) + + os.symlink("/Applications", os.path.join(disk_root, "Applications")) + + print("+ Finalizing .dmg disk image +") + + run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + + os.unlink(tempname) # ------------------------------------------------ -if verbose >= 2: - print("+ Done +") +print("+ Done +") sys.exit(0) diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index 502c20d0d6..3bca094d3b 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -16,6 +16,12 @@ The seeds compiled into the release are created from sipa's DNS seed data, like ## Dependencies -Ubuntu: +Ubuntu, Debian: sudo apt-get install python3-dnspython + +and/or for other operating systems: + + pip install dnspython + +See https://dnspython.readthedocs.io/en/latest/installation.html for more information. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 6d9d49ad2f..9be6a690a6 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -136,7 +136,7 @@ def lookup_asn(net, ip): ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 prefix = '.origin6' - asn = int([x.to_text() for x in dns.resolver.query('.'.join( + asn = int([x.to_text() for x in dns.resolver.resolve('.'.join( reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) return asn diff --git a/contrib/signet/README.md b/contrib/signet/README.md new file mode 100644 index 0000000000..71dc2f9638 --- /dev/null +++ b/contrib/signet/README.md @@ -0,0 +1,80 @@ +Contents +======== +This directory contains tools related to Signet, both for running a Signet yourself and for using one. + +getcoins.py +=========== + +A script to call a faucet to get Signet coins. + +Syntax: `getcoins.py [-h|--help] [-c|--cmd=<bitcoin-cli path>] [-f|--faucet=<faucet URL>] [-a|--addr=<signet bech32 address>] [-p|--password=<faucet password>] [--] [<bitcoin-cli args>]` + +* `--cmd` lets you customize the bitcoin-cli path. By default it will look for it in the PATH +* `--faucet` lets you specify which faucet to use; the faucet is assumed to be compatible with https://github.com/kallewoof/bitcoin-faucet +* `--addr` lets you specify a Signet address; by default, the address must be a bech32 address. This and `--cmd` above complement each other (i.e. you do not need `bitcoin-cli` if you use `--addr`) +* `--password` lets you specify a faucet password; this is handy if you are in a classroom and set up your own faucet for your students; (above faucet does not limit by IP when password is enabled) + +If using the default network, invoking the script with no arguments should be sufficient under normal +circumstances, but if multiple people are behind the same IP address, the faucet will by default only +accept one claim per day. See `--password` above. + +miner +===== + +To mine the first block in your custom chain, you can run: + + cd src/ + CLI="./bitcoin-cli -conf=mysignet.conf" + MINER="..contrib/signet/miner" + GRIND="./bitcoin-util grind" + ADDR=$($CLI -signet getnewaddress) + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --set-block-time=-1 + +This will mine a block with the current timestamp. If you want to backdate the chain, you can give a different timestamp to --set-block-time. + +You will then need to pick a difficulty target. Since signet chains are primarily protected by a signature rather than proof of work, there is no need to spend as much energy as possible mining, however you may wish to choose to spend more time than the absolute minimum. The calibrate subcommand can be used to pick a target, eg: + + $MINER calibrate --grind-cmd="$GRIND" + nbits=1e00f403 for 25s average mining time + +It defaults to estimating an nbits value resulting in 25s average time to find a block, but the --seconds parameter can be used to pick a different target, or the --nbits parameter can be used to estimate how long it will take for a given difficulty. + +Using the --ongoing parameter will then cause the signet miner to create blocks indefinitely. It will pick the time between blocks so that difficulty is adjusted to match the provided --nbits value. + + $MINER --cli="$CLI" generate --grind-cmd="$GRIND" --address="$ADDR" --nbits=1e00f403 --ongoing + +Other options +------------- + +The --debug and --quiet options are available to control how noisy the signet miner's output is. Note that the --debug, --quiet and --cli parameters must all appear before the subcommand (generate, calibrate, etc) if used. + +Instead of specifying --ongoing, you can specify --max-blocks=N to mine N blocks and stop. + +Instead of using a single address, a ranged descriptor may be provided instead (via the --descriptor parameter), with the reward for the block at height H being sent to the H'th address generated from the descriptor. + +Instead of calculating a specific nbits value, --min-nbits can be specified instead, in which case the mininmum signet difficulty will be targeted. + +By default, the signet miner mines blocks at fixed intervals with minimal variation. If you want blocks to appear more randomly, as they do in mainnet, specify the --poisson option. + +Using the --multiminer parameter allows mining to be distributed amongst multiple miners. For example, if you have 3 miners and want to share blocks between them, specify --multiminer=1/3 on one, --multiminer=2/3 on another, and --multiminer=3/3 on the last one. If you want one to do 10% of blocks and two others to do 45% each, --multiminer=1-10/100 on the first, and --multiminer=11-55 and --multiminer=56-100 on the others. Note that which miner mines which block is determined by the previous block hash, so occasional runs of one miner doing many blocks in a row is to be expected. + +When --multiminer is used, if a miner is down and does not mine a block within five minutes of when it is due, the other miners will automatically act as redundant backups ensuring the chain does not halt. The --backup-delay parameter can be used to change how long a given miner waits, allowing one to be the primary backup (after five minutes) and another to be the secondary backup (after six minutes, eg). + +The --standby-delay parameter can be used to make a backup miner that only mines if a block doesn't arrive on time. This can be combined with --multiminer if desired. Setting --standby-delay also prevents the first block from being mined immediately. + +Advanced usage +-------------- + +The process generate follows internally is to get a block template, convert that into a PSBT, sign the PSBT, move the signature from the signed PSBT into the block template's coinbase, grind proof of work for the block, and then submit the block to the network. + +These steps can instead be done explicitly: + + $CLI -signet getblocktemplate '{"rules": ["signet","segwit"]}' | + $MINER --cli="$CLI" genpsbt --address="$ADDR" | + $CLI -signet -stdin walletprocesspsbt | + jq -r .psbt | + $MINER --cli="$CLI" solvepsbt --grind-cmd="$GRIND" | + $CLI -signet -stdin submitblock + +This is intended to allow you to replace part of the pipeline for further experimentation, if desired. + diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py new file mode 100755 index 0000000000..691f0bb1b6 --- /dev/null +++ b/contrib/signet/getcoins.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import subprocess +import requests +import sys + +parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') +parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') +parser.add_argument('-f', '--faucet', dest='faucet', default='https://signetfaucet.com/claim', help='URL of the faucet') +parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') +parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') + +args = parser.parse_args() + +if args.addr == '': + if args.bitcoin_cli_args == []: + args.bitcoin_cli_args = ['-signet'] + # get address for receiving coins + try: + args.addr = subprocess.check_output([args.cmd] + args.bitcoin_cli_args + ['getnewaddress', 'faucet', 'bech32']).strip() + except FileNotFoundError: + print('The binary', args.cmd, 'could not be found.') + exit() + +data = {'address': args.addr, 'password': args.password} +try: + res = requests.post(args.faucet, data=data) +except: + print('Unexpected error when contacting faucet:', sys.exc_info()[0]) + exit() +print(res.text) diff --git a/contrib/signet/miner b/contrib/signet/miner new file mode 100755 index 0000000000..a3fba49d0e --- /dev/null +++ b/contrib/signet/miner @@ -0,0 +1,639 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import base64 +import json +import logging +import math +import os.path +import re +import struct +import sys +import time +import subprocess + +from binascii import unhexlify +from io import BytesIO + +PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) +sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) + +from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, ToHex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, uint256_from_str # noqa: E402 +from test_framework.script import CScriptOp # noqa: E402 + +logging.basicConfig( + format='%(asctime)s %(levelname)s %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S') + +SIGNET_HEADER = b"\xec\xc7\xda\xa2" +PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed +RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") + +# #### some helpers that could go into test_framework + +# like FromHex, but without the hex part +def FromBinary(cls, stream): + """deserialize a binary stream (or bytes object) into an object""" + # handle bytes object by turning it into a stream + was_bytes = isinstance(stream, bytes) + if was_bytes: + stream = BytesIO(stream) + obj = cls() + obj.deserialize(stream) + if was_bytes: + assert len(stream.read()) == 0 + return obj + +class PSBTMap: + """Class for serializing and deserializing PSBT maps""" + + def __init__(self, map=None): + self.map = map if map is not None else {} + + def deserialize(self, f): + m = {} + while True: + k = deser_string(f) + if len(k) == 0: + break + v = deser_string(f) + if len(k) == 1: + k = k[0] + assert k not in m + m[k] = v + self.map = m + + def serialize(self): + m = b"" + for k,v in self.map.items(): + if isinstance(k, int) and 0 <= k and k <= 255: + k = bytes([k]) + m += ser_compact_size(len(k)) + k + m += ser_compact_size(len(v)) + v + m += b"\x00" + return m + +class PSBT: + """Class for serializing and deserializing PSBTs""" + + def __init__(self): + self.g = PSBTMap() + self.i = [] + self.o = [] + self.tx = None + + def deserialize(self, f): + assert f.read(5) == b"psbt\xff" + self.g = FromBinary(PSBTMap, f) + assert 0 in self.g.map + self.tx = FromBinary(CTransaction, self.g.map[0]) + self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] + self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] + return self + + def serialize(self): + assert isinstance(self.g, PSBTMap) + assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) + assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) + assert 0 in self.g.map + tx = FromBinary(CTransaction, self.g.map[0]) + assert len(tx.vin) == len(self.i) + assert len(tx.vout) == len(self.o) + + psbt = [x.serialize() for x in [self.g] + self.i + self.o] + return b"psbt\xff" + b"".join(psbt) + + def to_base64(self): + return base64.b64encode(self.serialize()).decode("utf8") + + @classmethod + def from_base64(cls, b64psbt): + return FromBinary(cls, base64.b64decode(b64psbt)) + +# ##### + +def create_coinbase(height, value, spk): + cb = CTransaction() + cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] + cb.vout = [CTxOut(value, spk)] + return cb + +def get_witness_script(witness_root, witness_nonce): + commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) + return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) + +def signet_txs(block, challenge): + # assumes signet solution has not been added yet so does not need + # to be removed + + txs = block.vtx[:] + txs[0] = CTransaction(txs[0]) + txs[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER) + hashes = [] + for tx in txs: + tx.rehash() + hashes.append(ser_uint256(tx.sha256)) + mroot = block.get_merkle_root(hashes) + + sd = b"" + sd += struct.pack("<i", block.nVersion) + sd += ser_uint256(block.hashPrevBlock) + sd += ser_uint256(mroot) + sd += struct.pack("<I", block.nTime) + + to_spend = CTransaction() + to_spend.nVersion = 0 + to_spend.nLockTime = 0 + to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] + to_spend.vout = [CTxOut(0, challenge)] + to_spend.rehash() + + spend = CTransaction() + spend.nVersion = 0 + spend.nLockTime = 0 + spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] + spend.vout = [CTxOut(0, b"\x6a")] + + return spend, to_spend + +def do_createpsbt(block, signme, spendme): + psbt = PSBT() + psbt.g = PSBTMap( {0: signme.serialize(), + PSBT_SIGNET_BLOCK: block.serialize() + } ) + psbt.i = [ PSBTMap( {0: spendme.serialize(), + 3: bytes([1,0,0,0])}) + ] + psbt.o = [ PSBTMap() ] + return psbt.to_base64() + +def do_decode_psbt(b64psbt): + psbt = PSBT.from_base64(b64psbt) + + assert len(psbt.tx.vin) == 1 + assert len(psbt.tx.vout) == 1 + assert PSBT_SIGNET_BLOCK in psbt.g.map + + scriptSig = psbt.i[0].map.get(7, b"") + scriptWitness = psbt.i[0].map.get(8, b"\x00") + + return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + +def finish_block(block, signet_solution, grind_cmd): + block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) + block.vtx[0].rehash() + block.hashMerkleRoot = block.calc_merkle_root() + if grind_cmd is None: + block.solve() + else: + headhex = CBlockHeader.serialize(block).hex() + cmd = grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + newhead = FromHex(CBlockHeader(), newheadhex.decode('utf8')) + block.nNonce = newhead.nNonce + block.rehash() + return block + +def generate_psbt(tmpl, reward_spk, *, blocktime=None): + signet_spk = tmpl["signet_challenge"] + signet_spk_bin = unhexlify(signet_spk) + + cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) + cbtx.vin[0].nSequence = 2**32-2 + cbtx.rehash() + + block = CBlock() + block.nVersion = tmpl["version"] + block.hashPrevBlock = int(tmpl["previousblockhash"], 16) + block.nTime = tmpl["curtime"] if blocktime is None else blocktime + if block.nTime < tmpl["mintime"]: + block.nTime = tmpl["mintime"] + block.nBits = int(tmpl["bits"], 16) + block.nNonce = 0 + block.vtx = [cbtx] + [FromHex(CTransaction(), t["data"]) for t in tmpl["transactions"]] + + witnonce = 0 + witroot = block.calc_witness_merkle_root() + cbwit = CTxInWitness() + cbwit.scriptWitness.stack = [ser_uint256(witnonce)] + block.vtx[0].wit.vtxinwit = [cbwit] + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + + signme, spendme = signet_txs(block, signet_spk_bin) + + return do_createpsbt(block, signme, spendme) + +def get_reward_address(args, height): + if args.address is not None: + return args.address + + if '*' not in args.descriptor: + addr = json.loads(args.bcli("deriveaddresses", args.descriptor))[0] + args.address = addr + return addr + + remove = [k for k in args.derived_addresses.keys() if k+20 <= height] + for k in remove: + del args.derived_addresses[k] + + addr = args.derived_addresses.get(height, None) + if addr is None: + addrs = json.loads(args.bcli("deriveaddresses", args.descriptor, "[%d,%d]" % (height, height+20))) + addr = addrs[0] + for k, a in enumerate(addrs): + args.derived_addresses[height+k] = a + + return addr + +def get_reward_addr_spk(args, height): + assert args.address is not None or args.descriptor is not None + + if hasattr(args, "reward_spk"): + return args.address, args.reward_spk + + reward_addr = get_reward_address(args, height) + reward_spk = unhexlify(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + if args.address is not None: + # will always be the same, so cache + args.reward_spk = reward_spk + + return reward_addr, reward_spk + +def do_genpsbt(args): + tmpl = json.load(sys.stdin) + _, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + psbt = generate_psbt(tmpl, reward_spk) + print(psbt) + +def do_solvepsbt(args): + block, signet_solution = do_decode_psbt(sys.stdin.read()) + block = finish_block(block, signet_solution, args.grind_cmd) + print(ToHex(block)) + +def nbits_to_target(nbits): + shift = (nbits >> 24) & 0xff + return (nbits & 0x00ffffff) * 2**(8*(shift - 3)) + +def target_to_nbits(target): + tstr = "{0:x}".format(target) + if len(tstr) < 6: + tstr = ("000000"+tstr)[-6:] + if len(tstr) % 2 != 0: + tstr = "0" + tstr + if int(tstr[0],16) >= 0x8: + # avoid "negative" + tstr = "00" + tstr + fix = int(tstr[:6], 16) + sz = len(tstr)//2 + if tstr[6:] != "0"*(sz*2-6): + fix += 1 + + return int("%02x%06x" % (sz,fix), 16) + +def seconds_to_hms(s): + if s == 0: + return "0s" + neg = (s < 0) + if neg: + s = -s + out = "" + if s % 60 > 0: + out = "%ds" % (s % 60) + s //= 60 + if s % 60 > 0: + out = "%dm%s" % (s % 60, out) + s //= 60 + if s > 0: + out = "%dh%s" % (s, out) + if neg: + out = "-" + out + return out + +def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): + # strategy: + # 1) work out how far off our desired target we are + # 2) cap it to a factor of 4 since that's the best we can do in a single retarget period + # 3) use that to work out the desired average interval in this retarget period + # 4) if doing poisson, use the last hash to pick a uniformly random number in [0,1), and work out a random multiplier to vary the average by + # 5) cap the resulting interval between 1 second and 1 hour to avoid extremes + + INTERVAL = 600.0*2016/2015 # 10 minutes, adjusted for the off-by-one bug + + current_target = nbits_to_target(last_nbits) + retarget_factor = ultimate_target / current_target + retarget_factor = max(0.25, min(retarget_factor, 4.0)) + + avg_interval = INTERVAL * retarget_factor + + if do_poisson: + det_rand = int(last_hash[-8:], 16) * 2**-32 + this_interval_variance = -math.log1p(-det_rand) + else: + this_interval_variance = 1 + + this_interval = avg_interval * this_interval_variance + this_interval = max(1, min(this_interval, 3600)) + + return this_interval + +def next_block_is_mine(last_hash, my_blocks): + det_rand = int(last_hash[-16:-8], 16) + return my_blocks[0] <= (det_rand % my_blocks[2]) < my_blocks[1] + +def do_generate(args): + if args.max_blocks is not None: + if args.ongoing: + logging.error("Cannot specify both --ongoing and --max-blocks") + return 1 + if args.max_blocks < 1: + logging.error("N must be a positive integer") + return 1 + max_blocks = args.max_blocks + elif args.ongoing: + max_blocks = None + else: + max_blocks = 1 + + if args.set_block_time is not None and max_blocks != 1: + logging.error("Cannot specify --ongoing or --max-blocks > 1 when using --set-block-time") + return 1 + if args.set_block_time is not None and args.set_block_time < 0: + args.set_block_time = time.time() + logging.info("Treating negative block time as current time (%d)" % (args.set_block_time)) + + if args.min_nbits: + if args.nbits is not None: + logging.error("Cannot specify --nbits and --min-nbits") + return 1 + args.nbits = "1e0377ae" + logging.info("Using nbits=%s" % (args.nbits)) + + if args.set_block_time is None: + if args.nbits is None or len(args.nbits) != 8: + logging.error("Must specify --nbits (use calibrate command to determine value)") + return 1 + + if args.multiminer is None: + my_blocks = (0,1,1) + else: + if not args.ongoing: + logging.error("Cannot specify --multiminer without --ongoing") + return 1 + m = RE_MULTIMINER.match(args.multiminer) + if m is None: + logging.error("--multiminer argument must be k/m or j-k/m") + return 1 + start,_,stop,total = m.groups() + if stop is None: + stop = start + start, stop, total = map(int, (start, stop, total)) + if stop < start or start <= 0 or total < stop or total == 0: + logging.error("Inconsistent values for --multiminer") + return 1 + my_blocks = (start-1, stop, total) + + ultimate_target = nbits_to_target(int(args.nbits,16)) + + mined_blocks = 0 + bestheader = {"hash": None} + lastheader = None + while max_blocks is None or mined_blocks < max_blocks: + + # current status? + bci = json.loads(args.bcli("getblockchaininfo")) + + if bestheader["hash"] != bci["bestblockhash"]: + bestheader = json.loads(args.bcli("getblockheader", bci["bestblockhash"])) + + if lastheader is None: + lastheader = bestheader["hash"] + elif bestheader["hash"] != lastheader: + next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson) + next_delta += bestheader["time"] - time.time() + next_is_mine = next_block_is_mine(bestheader["hash"], my_blocks) + logging.info("Received new block at height %d; next in %s (%s)", bestheader["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + lastheader = bestheader["hash"] + + # when is the next block due to be mined? + now = time.time() + if args.set_block_time is not None: + logging.debug("Setting start time to %d", args.set_block_time) + mine_time = args.set_block_time + action_time = now + is_mine = True + elif bestheader["height"] == 0: + logging.error("When mining first block in a new signet, must specify --set-block-time") + return 1 + else: + + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + mine_time = bestheader["time"] + time_delta + + is_mine = next_block_is_mine(bci["bestblockhash"], my_blocks) + + action_time = mine_time + if not is_mine: + action_time += args.backup_delay + + if args.standby_delay > 0: + action_time += args.standby_delay + elif mined_blocks == 0: + # for non-standby, always mine immediately on startup, + # even if the next block shouldn't be ours + action_time = now + + # don't want fractional times so round down + mine_time = int(mine_time) + action_time = int(action_time) + + # can't mine a block 2h in the future; 1h55m for some safety + action_time = max(action_time, mine_time - 6900) + + # ready to go? otherwise sleep and check for new block + if now < action_time: + sleep_for = min(action_time - now, 60) + if mine_time < now: + # someone else might have mined the block, + # so check frequently, so we don't end up late + # mining the next block if it's ours + sleep_for = min(20, sleep_for) + minestr = "mine" if is_mine else "backup" + logging.debug("Sleeping for %s, next block due in %s (%s)" % (seconds_to_hms(sleep_for), seconds_to_hms(mine_time - now), minestr)) + time.sleep(sleep_for) + continue + + # gbt + tmpl = json.loads(args.bcli("getblocktemplate", '{"rules":["signet","segwit"]}')) + if tmpl["previousblockhash"] != bci["bestblockhash"]: + logging.warning("GBT based off unexpected block (%s not %s), retrying", tmpl["previousblockhash"], bci["bestblockhash"]) + time.sleep(1) + continue + + logging.debug("GBT template: %s", tmpl) + + if tmpl["mintime"] > mine_time: + logging.info("Updating block time from %d to %d", mine_time, tmpl["mintime"]) + mine_time = tmpl["mintime"] + if mine_time > now: + logging.error("GBT mintime is in the future: %d is %d seconds later than %d", mine_time, (mine_time-now), now) + return 1 + + # address for reward + reward_addr, reward_spk = get_reward_addr_spk(args, tmpl["height"]) + + # mine block + logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) + mined_blocks += 1 + psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + if not psbt_signed.get("complete",False): + logging.debug("Generated PSBT: %s" % (psbt,)) + sys.stderr.write("PSBT signing failed") + return 1 + block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) + block = finish_block(block, signet_solution, args.grind_cmd) + + # submit block + r = args.bcli("-stdin", "submitblock", input=ToHex(block).encode('utf8')) + + # report + bstr = "block" if is_mine else "backup block" + + next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson) + next_delta += block.nTime - time.time() + next_is_mine = next_block_is_mine(block.hash, my_blocks) + + logging.debug("Block hash %s payout to %s", block.hash, reward_addr) + logging.info("Mined %s at height %d; next in %s (%s)", bstr, tmpl["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) + if r != "": + logging.warning("submitblock returned %s for height %d hash %s", r, tmpl["height"], block.hash) + lastheader = block.hash + +def do_calibrate(args): + if args.nbits is not None and args.seconds is not None: + sys.stderr.write("Can only specify one of --nbits or --seconds\n") + return 1 + if args.nbits is not None and len(args.nbits) != 8: + sys.stderr.write("Must specify 8 hex digits for --nbits") + return 1 + + TRIALS = 600 # gets variance down pretty low + TRIAL_BITS = 0x1e3ea75f # takes about 5m to do 600 trials + #TRIAL_BITS = 0x1e7ea75f # XXX + + header = CBlockHeader() + header.nBits = TRIAL_BITS + targ = nbits_to_target(header.nBits) + + start = time.time() + count = 0 + #CHECKS=[] + for i in range(TRIALS): + header.nTime = i + header.nNonce = 0 + headhex = header.serialize().hex() + cmd = args.grind_cmd.split(" ") + [headhex] + newheadhex = subprocess.run(cmd, stdout=subprocess.PIPE, input=b"", check=True).stdout.strip() + #newhead = FromHex(CBlockHeader(), newheadhex.decode('utf8')) + #count += newhead.nNonce + #if (i+1) % 100 == 0: + # CHECKS.append((i+1, count, time.time()-start)) + + #print("checks =", [c*1.0 / (b*targ*2**-256) for _,b,c in CHECKS]) + + avg = (time.time() - start) * 1.0 / TRIALS + #exp_count = 2**256 / targ * TRIALS + #print("avg =", avg, "count =", count, "exp_count =", exp_count) + + if args.nbits is not None: + want_targ = nbits_to_target(int(args.nbits,16)) + want_time = avg*targ/want_targ + else: + want_time = args.seconds if args.seconds is not None else 25 + want_targ = int(targ*(avg/want_time)) + + print("nbits=%08x for %ds average mining time" % (target_to_nbits(want_targ), want_time)) + return 0 + +def bitcoin_cli(basecmd, args, **kwargs): + cmd = basecmd + ["-signet"] + args + logging.debug("Calling bitcoin-cli: %r", cmd) + out = subprocess.run(cmd, stdout=subprocess.PIPE, **kwargs, check=True).stdout + if isinstance(out, bytes): + out = out.decode('utf8') + return out.strip() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cli", default="bitcoin-cli", type=str, help="bitcoin-cli command") + parser.add_argument("--debug", action="store_true", help="Print debugging info") + parser.add_argument("--quiet", action="store_true", help="Only print warnings/errors") + + cmds = parser.add_subparsers(help="sub-commands") + genpsbt = cmds.add_parser("genpsbt", help="Generate a block PSBT for signing") + genpsbt.set_defaults(fn=do_genpsbt) + + solvepsbt = cmds.add_parser("solvepsbt", help="Solve a signed block PSBT") + solvepsbt.set_defaults(fn=do_solvepsbt) + + generate = cmds.add_parser("generate", help="Mine blocks") + generate.set_defaults(fn=do_generate) + generate.add_argument("--ongoing", action="store_true", help="Keep mining blocks") + generate.add_argument("--max-blocks", default=None, type=int, help="Max blocks to mine (default=1)") + generate.add_argument("--set-block-time", default=None, type=int, help="Set block time (unix timestamp)") + generate.add_argument("--nbits", default=None, type=str, help="Target nBits (specify difficulty)") + generate.add_argument("--min-nbits", action="store_true", help="Target minimum nBits (use min difficulty)") + generate.add_argument("--poisson", action="store_true", help="Simulate randomised block times") + #generate.add_argument("--signcmd", default=None, type=str, help="Alternative signing command") + generate.add_argument("--multiminer", default=None, type=str, help="Specify which set of blocks to mine (eg: 1-40/100 for the first 40%%, 2/3 for the second 3rd)") + generate.add_argument("--backup-delay", default=300, type=int, help="Seconds to delay before mining blocks reserved for other miners (default=300)") + generate.add_argument("--standby-delay", default=0, type=int, help="Seconds to delay before mining blocks (default=0)") + + calibrate = cmds.add_parser("calibrate", help="Calibrate difficulty") + calibrate.set_defaults(fn=do_calibrate) + calibrate.add_argument("--nbits", type=str, default=None) + calibrate.add_argument("--seconds", type=int, default=None) + + for sp in [genpsbt, generate]: + sp.add_argument("--address", default=None, type=str, help="Address for block reward payment") + sp.add_argument("--descriptor", default=None, type=str, help="Descriptor for block reward payment") + + for sp in [solvepsbt, generate, calibrate]: + sp.add_argument("--grind-cmd", default=None, type=str, help="Command to grind a block header for proof-of-work") + + args = parser.parse_args(sys.argv[1:]) + + args.bcli = lambda *a, input=b"", **kwargs: bitcoin_cli(args.cli.split(" "), list(a), input=input, **kwargs) + + if hasattr(args, "address") and hasattr(args, "descriptor"): + if args.address is None and args.descriptor is None: + sys.stderr.write("Must specify --address or --descriptor\n") + return 1 + elif args.address is not None and args.descriptor is not None: + sys.stderr.write("Only specify one of --address or --descriptor\n") + return 1 + args.derived_addresses = {} + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + elif args.quiet: + logging.getLogger().setLevel(logging.WARNING) + else: + logging.getLogger().setLevel(logging.INFO) + + if hasattr(args, "fn"): + return args.fn(args) + else: + logging.error("Must specify command") + return 1 + +if __name__ == "__main__": + main() + + diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py index c7ebac50d4..87341ccf96 100644 --- a/contrib/testgen/base58.py +++ b/contrib/testgen/base58.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012-2018 The Bitcoin Core developers +# Copyright (c) 2012-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index 49320d92e6..8a3918da6b 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2018 The Bitcoin Core developers +# Copyright (c) 2012-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 8b8503331d..9cb887e2dc 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/depends/Makefile b/depends/Makefile index 1ad21f6821..596a46d4a2 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -37,6 +37,7 @@ NO_QR ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= +NO_NATPMP ?= MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources @@ -126,6 +127,8 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# Make sure that cache is invalidated when switching between system and +# depends-managed, pinned clang build_id_string+=system_clang $(host_arch)_$(host_os)_id_string+=system_clang endif @@ -139,10 +142,12 @@ sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) upnp_packages_$(NO_UPNP) = $(upnp_packages) +natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) + zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -163,12 +168,6 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk -binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin) -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) -else -toolchain_path= -endif final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -179,15 +178,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ +# $PATH is not preserved between ./configure and make by convention. Its +# modification and overriding at ./configure time is (as I understand it) +# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros, +# which will expand the program names to their full absolute paths. The notable +# exception is command line overriding: ./configure CC=clang, which skips the +# program name expansion step, and works because the user implicitly indicates +# with CC=clang that clang will be available in $PATH at all times, and is most +# likely part of the user's system. +# +# Therefore, when we "seed the autoconf cache"/"override well-known program +# vars" by setting AR=<blah> in our config.site, either one of two things needs +# to be true for the build system to work correctly: +# +# 1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the +# tool needs to be available in $PATH at all times. +# +# 2. If the tool is _**not**_ expected to be available in $PATH at all times +# (such as is the case for our native_cctools binutils tools), it needs to +# be referred to by its absolute path, such as would be output by the +# AC_PATH_{PROG,TOOL} macros. +# +# Minor note: it is also okay to refer to tools by their absolute path even if +# we expect them to be available in $PATH at all times, more specificity does +# not hurt. $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(binutils_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(binutils_path)$(host_NM)|' \ - -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \ + -e 's|@CC@|$(host_CC)|' \ + -e 's|@CXX@|$(host_CXX)|' \ + -e 's|@AR@|$(host_AR)|' \ + -e 's|@RANLIB@|$(host_RANLIB)|' \ + -e 's|@NM@|$(host_NM)|' \ + -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ @@ -200,6 +223,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ + -e 's|@no_natpmp@|$(NO_NATPMP)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ diff --git a/depends/README.md b/depends/README.md index 5225a6d5c4..9bc9b6714b 100644 --- a/depends/README.md +++ b/depends/README.md @@ -25,7 +25,7 @@ Common `host-platform-triplets` for cross compilation are: - `i686-pc-linux-gnu` for Linux 32 bit - `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin16` for macOS +- `x86_64-apple-darwin18` for macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) @@ -80,48 +80,31 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu ### Dependency Options + The following can be set when running make: `make FOO=bar` -<dl> -<dt>SOURCES_PATH</dt> -<dd>downloaded sources will be placed here</dd> -<dt>BASE_CACHE</dt> -<dd>built packages will be placed here</dd> -<dt>SDK_PATH</dt> -<dd>Path where sdk's can be found (used by macOS)</dd> -<dt>FALLBACK_DOWNLOAD_PATH</dt> -<dd>If a source file can't be fetched, try here before giving up</dd> -<dt>NO_QT</dt> -<dd>Don't download/build/cache qt and its dependencies</dd> -<dt>NO_QR</dt> -<dd>Don't download/build/cache packages needed for enabling qrencode</dd> -<dt>NO_ZMQ</dt> -<dd>Don't download/build/cache packages needed for enabling zeromq</dd> -<dt>NO_WALLET</dt> -<dd>Don't download/build/cache libs needed to enable the wallet</dd> -<dt>NO_BDB</dt> -<dd>Don't download/build/cache BerkeleyDB</dd> -<dt>NO_SQLITE</dt> -<dd>Don't download/build/cache SQLite</dd> -<dt>NO_UPNP</dt> -<dd>Don't download/build/cache packages needed for enabling upnp</dd> -<dt>ALLOW_HOST_PACKAGES</dt> -<dd>Packages that are missed in dependencies (due to `NO_*` option or -build script logic) are searched for among the host system packages using -`pkg-config`. It allows building with packages of other (newer) versions</dd> -<dt>MULTIPROCESS</dt> -<dd>build libmultiprocess (experimental, requires cmake)</dd> -<dt>DEBUG</dt> -<dd>disable some optimizations and enable more runtime checking</dd> -<dt>HOST_ID_SALT</dt> -<dd>Optional salt to use when generating host package ids</dd> -<dt>BUILD_ID_SALT</dt> -<dd>Optional salt to use when generating build package ids</dd> -<dt>FORCE_USE_SYSTEM_CLANG</dt> -<dd>(EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the -system's <code>$PATH</code> rather than the default prebuilt release of Clang -from llvm.org. Clang 8 or later is required.</dd> -</dl> +- `SOURCES_PATH`: Downloaded sources will be placed here +- `BASE_CACHE`: Built packages will be placed here +- `SDK_PATH`: Path where SDKs can be found (used by macOS) +- `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up +- `NO_QT`: Don't download/build/cache Qt and its dependencies +- `NO_QR`: Don't download/build/cache packages needed for enabling qrencode +- `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ +- `NO_WALLET`: Don't download/build/cache libs needed to enable the wallet +- `NO_BDB`: Don't download/build/cache BerkeleyDB +- `NO_SQLITE`: Don't download/build/cache SQLite +- `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP +- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP</dd> +- `ALLOW_HOST_PACKAGES`: Packages that are missed in dependencies (due to `NO_*` option or + build script logic) are searched for among the host system packages using + `pkg-config`. It allows building with packages of other (newer) versions +- `MULTIPROCESS`: Build libmultiprocess (experimental, requires CMake) +- `DEBUG`: Disable some optimizations and enable more runtime checking +- `HOST_ID_SALT`: Optional salt to use when generating host package ids +- `BUILD_ID_SALT`: Optional salt to use when generating build package ids +- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the + system's `$PATH` rather than the default prebuilt release of Clang + from llvm.org. Clang 8 or later is required. If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/config.site.in b/depends/config.site.in index 87a3379303..391767357a 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -1,67 +1,81 @@ -depends_prefix="`dirname ${ac_site_file}`/.." +# shellcheck shell=sh disable=SC2034 # Many variables set will be used in + # ./configure but shellcheck doesn't know + # that, hence: disable=SC2034 + +true # Dummy command because shellcheck treats all directives before first + # command as file-wide, and we only want to disable for one line. + # + # See: https://github.com/koalaman/shellcheck/wiki/Directive + +# shellcheck disable=SC2154 +depends_prefix="$(cd "$(dirname "$ac_site_file")/.." && pwd)" cross_compiling=maybe -host_alias=@HOST@ -ac_tool_prefix=${host_alias}- +host_alias="@HOST@" +ac_tool_prefix="${host_alias}-" -if test -z $with_boost; then - with_boost=$depends_prefix +if test -z "$with_boost"; then + with_boost="$depends_prefix" fi -if test -z $with_qt_plugindir; then - with_qt_plugindir=$depends_prefix/plugins +if test -z "$with_qt_plugindir"; then + with_qt_plugindir="${depends_prefix}/plugins" fi -if test -z $with_qt_translationdir; then - with_qt_translationdir=$depends_prefix/translations +if test -z "$with_qt_translationdir"; then + with_qt_translationdir="${depends_prefix}/translations" fi -if test -z $with_qt_bindir && test -z "@no_qt@"; then - with_qt_bindir=$depends_prefix/native/bin +if test -z "$with_qt_bindir" && test -z "@no_qt@"; then + with_qt_bindir="${depends_prefix}/native/bin" fi -if test -z $with_mpgen && test -n "@multiprocess@"; then - with_mpgen=$depends_prefix/native +if test -z "$with_mpgen" && test -n "@multiprocess@"; then + with_mpgen="${depends_prefix}/native" fi -if test -z $with_qrencode && test -n "@no_qr@"; then +if test -z "$with_qrencode" && test -n "@no_qr@"; then with_qrencode=no fi -if test -z $enable_wallet && test -n "@no_wallet@"; then +if test -z "$enable_wallet" && test -n "@no_wallet@"; then enable_wallet=no fi -if test -z $enable_multiprocess && test -n "@multiprocess@"; then +if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then enable_multiprocess=yes fi -if test -z $with_miniupnpc && test -n "@no_upnp@"; then +if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then with_miniupnpc=no fi -if test -z $with_gui && test -n "@no_qt@"; then +if test -z "$with_natpmp" && test -n "@no_natpmp@"; then + with_natpmp=no +fi + +if test -z "$with_gui" && test -n "@no_qt@"; then with_gui=no fi -if test -z $enable_zmq && test -n "@no_zmq@"; then +if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi -if test x@host_os@ = xdarwin; then +if test "x@host_os@" = xdarwin; then BREW=no PORT=no fi -PATH=$depends_prefix/native/bin:$PATH -PKG_CONFIG="`which pkg-config` --static" +PATH="${depends_prefix}/native/bin:${PATH}" +PKG_CONFIG="$(which pkg-config) --static" # These two need to remain exported because pkg-config does not see them # otherwise. That means they must be unexported at the end of configure.ac to # avoid ruining the cache. Sigh. -export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig:$depends_prefix/lib/pkgconfig +export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" if test -z "@allow_host_packages@"; then - export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig + export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" fi -CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" -LDFLAGS="-L$depends_prefix/lib $LDFLAGS" +CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}" +LDFLAGS="-L${depends_prefix}/lib ${LDFLAGS}" if test -n "@CC@" -a -z "${CC}"; then CC="@CC@" @@ -69,21 +83,21 @@ fi if test -n "@CXX@" -a -z "${CXX}"; then CXX="@CXX@" fi -PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH +PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}" if test -n "@AR@"; then - AR=@AR@ - ac_cv_path_ac_pt_AR=${AR} + AR="@AR@" + ac_cv_path_ac_pt_AR="${AR}" fi if test -n "@RANLIB@"; then - RANLIB=@RANLIB@ - ac_cv_path_ac_pt_RANLIB=${RANLIB} + RANLIB="@RANLIB@" + ac_cv_path_ac_pt_RANLIB="${RANLIB}" fi if test -n "@NM@"; then - NM=@NM@ - ac_cv_path_ac_pt_NM=${NM} + NM="@NM@" + ac_cv_path_ac_pt_NM="${NM}" fi if test -n "@debug@"; then @@ -91,14 +105,14 @@ if test -n "@debug@"; then fi if test -n "@CFLAGS@"; then - CFLAGS="@CFLAGS@ $CFLAGS" + CFLAGS="@CFLAGS@ ${CFLAGS}" fi if test -n "@CXXFLAGS@"; then - CXXFLAGS="@CXXFLAGS@ $CXXFLAGS" + CXXFLAGS="@CXXFLAGS@ ${CXXFLAGS}" fi if test -n "@CPPFLAGS@"; then - CPPFLAGS="@CPPFLAGS@ $CPPFLAGS" + CPPFLAGS="@CPPFLAGS@ ${CPPFLAGS}" fi if test -n "@LDFLAGS@"; then - LDFLAGS="@LDFLAGS@ $LDFLAGS" + LDFLAGS="@LDFLAGS@ ${LDFLAGS}" fi diff --git a/depends/funcs.mk b/depends/funcs.mk index 58d882eb05..c0159f0e38 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -1,17 +1,23 @@ define int_vars #Set defaults for vars which may be overridden per-package -$(1)_cc=$($($(1)_type)_CC) -$(1)_cxx=$($($(1)_type)_CXX) -$(1)_objc=$($($(1)_type)_OBJC) -$(1)_objcxx=$($($(1)_type)_OBJCXX) -$(1)_ar=$($($(1)_type)_AR) -$(1)_ranlib=$($($(1)_type)_RANLIB) -$(1)_libtool=$($($(1)_type)_LIBTOOL) -$(1)_nm=$($($(1)_type)_NM) -$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) -$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) -$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib -$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_cc=$$($$($(1)_type)_CC) +$(1)_cxx=$$($$($(1)_type)_CXX) +$(1)_objc=$$($$($(1)_type)_OBJC) +$(1)_objcxx=$$($$($(1)_type)_OBJCXX) +$(1)_ar=$$($$($(1)_type)_AR) +$(1)_ranlib=$$($$($(1)_type)_RANLIB) +$(1)_libtool=$$($$($(1)_type)_LIBTOOL) +$(1)_nm=$$($$($(1)_type)_NM) +$(1)_cflags=$$($$($(1)_type)_CFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CFLAGS) +$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CXXFLAGS) +$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \ + $$($$($(1)_type)_$$(release_type)_LDFLAGS) \ + -L$$($($(1)_type)_prefix)/lib +$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \ + -I$$($$($(1)_type)_prefix)/include $(1)_recipe_hash:= endef @@ -163,7 +169,9 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" -ifneq ($($(1)_type),build) +ifeq ($($(1)_type),build) +$(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" +else ifneq ($(host),$(build)) $(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) $(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 6099fd4c71..646e97837a 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=10.12 +OSX_MIN_VERSION=10.14 OSX_SDK_VERSION=10.15.1 XCODE_VERSION=11.3.1 XCODE_BUILD_ID=11C505 @@ -6,6 +6,49 @@ LD64_VERSION=530 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers +darwin_native_binutils=native_cctools + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang +# from llvm.org + +# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG +# is empty +darwin_native_toolchain=native_cctools + +clang_prog=$(build_prefix)/bin/clang +clangxx_prog=$(clang_prog)++ + +clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version) +else +# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's +# system + +darwin_native_toolchain= + +# We can't just use $(shell command -v clang) because GNU Make handles builtins +# in a special way and doesn't know that `command` is a POSIX-standard builtin +# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. +# At the time of writing, GNU Make v4.2.1 is still being used in supported +# distro releases. +# +# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +clang_resource_dir=$(shell clang -print-resource-dir) +endif + +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL + +# Make-only lowercase function +lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) + +# For well-known tools provided by cctools, make sure that their well-known +# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} +# would. +$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL)))) + # Flag explanations: # # -mlinker-version @@ -18,7 +61,7 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with- # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 +# -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 # # Forces clang to use the libc++ headers from our SDK and completely # forget about the libc++ headers from the standard directories @@ -28,8 +71,49 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with- # https://reviews.llvm.org/D64089, we should use that instead. Read the # differential summary there for more details. # -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 +# -Xclang -*system<path_a> \ +# -Xclang -*system<path_b> \ +# -Xclang -*system<path_c> ... +# +# Adds path_a, path_b, and path_c to the bottom of clang's list of +# include search paths. This is used to explicitly specify the list of +# system include search paths and its ordering, rather than rely on +# clang's autodetection routine. This routine has been shown to: +# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 +# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) +# 2. Fail to pickup C headers in $SYSROOT/usr/include when +# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) +# +# Talking directly to cc1 with -Xclang here grants us access to specify +# more granular categories for these system include search paths, and we +# can use the correct categories that these search paths would have been +# placed in if the autodetection routine had worked correctly. (see: +# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) +# +# Furthermore, it places these search paths after any "non-Xclang" +# specified search paths. This prevents any additional clang options or +# environment variables from coming after or in between these system +# include search paths, as that would be wrong in general but would also +# break #include_next's. +# +darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include +darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -stdlib=libc++ -nostdinc++ \ + -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -40,11 +124,4 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) -darwin_native_binutils=native_cctools -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_toolchain=native_cctools -else -darwin_native_toolchain= -endif - darwin_cmake_system=Darwin diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 5953341d9f..d45ac3d03f 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -10,8 +10,9 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic +$(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -$(package)_cxxflags=-std=c++11 +$(package)_cxxflags=-std=c++17 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index d8bce108b1..ff8a252db9 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,50 +1,45 @@ package=boost -$(package)_version=1_70_0 -$(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ +$(package)_version=1_71_0 +$(package)_download_path=https://dl.bintray.com/boostorg/release/$(subst _,.,$($(package)_version))/source/ $(package)_file_name=boost_$($(package)_version).tar.bz2 -$(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 -$(package)_patches=unused_var_in_process.patch +$(package)_sha256_hash=d73a8da01e8bf8c7eda40b4c84915071a8c8a0df4a6734537ddde4a8580524ee +$(package)_dependencies=native_b2 define $(package)_set_vars $(package)_config_opts_release=variant=release $(package)_config_opts_debug=variant=debug $(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam -$(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1 +$(package)_config_opts+=threading=multi link=static -sNO_COMPRESSION=1 $(package)_config_opts_linux=target-os=linux threadapi=pthread runtime-link=shared $(package)_config_opts_darwin=target-os=darwin runtime-link=shared $(package)_config_opts_mingw32=target-os=windows binary-format=pe threadapi=win32 runtime-link=static -$(package)_config_opts_x86_64_mingw32=address-model=64 -$(package)_config_opts_i686_mingw32=address-model=32 -$(package)_config_opts_i686_linux=address-model=32 architecture=x86 -$(package)_config_opts_i686_android=address-model=32 -$(package)_config_opts_aarch64_android=address-model=64 -$(package)_config_opts_x86_64_android=address-model=64 -$(package)_config_opts_armv7a_android=address-model=32 -$(package)_toolset_$(host_os)=gcc -$(package)_toolset_darwin=clang +$(package)_config_opts_x86_64=architecture=x86 address-model=64 +$(package)_config_opts_i686=architecture=x86 address-model=32 +$(package)_config_opts_aarch64=address-model=64 +$(package)_config_opts_armv7a=address-model=32 ifneq (,$(findstring clang,$($(package)_cxx))) - $(package)_toolset_$(host_os)=clang +$(package)_toolset_$(host_os)=clang +else +$(package)_toolset_$(host_os)=gcc endif -$(package)_archiver_$(host_os)=$($(package)_ar) $(package)_config_libraries=filesystem,system,thread,test -$(package)_cxxflags=-std=c++11 -fvisibility=hidden +$(package)_cxxflags=-std=c++17 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/unused_var_in_process.patch && \ - echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam + echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cflags>\"$($(package)_cflags)\" <cxxflags>\"$($(package)_cxxflags)\" <compileflags>\"$($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_ar)\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) + ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) --with-bjam=b2 endef define $(package)_build_cmds - ./b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) stage + b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) stage endef define $(package)_stage_cmds - ./b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install + b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install endef diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk new file mode 100644 index 0000000000..a24f201859 --- /dev/null +++ b/depends/packages/libnatpmp.mk @@ -0,0 +1,19 @@ +package=libnatpmp +$(package)_version=20150609 +$(package)_download_path=https://miniupnp.tuxfamily.org/files/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=e1aa9c4c4219bc06943d6b2130f664daee213fb262fcb94dd355815b8f4536b0 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" +endef + +define $(package)_build_cmds + $(MAKE) libnatpmp.a $($(package)_build_opts) +endef + +define $(package)_stage_cmds + mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib &&\ + install *.h $($(package)_staging_prefix_dir)/include &&\ + install libnatpmp.a $($(package)_staging_prefix_dir)/lib +endef diff --git a/depends/packages/native_b2.mk b/depends/packages/native_b2.mk new file mode 100644 index 0000000000..aaa37cdcfa --- /dev/null +++ b/depends/packages/native_b2.mk @@ -0,0 +1,20 @@ +package=native_b2 +$(package)_version=$(boost_version) +$(package)_download_path=$(boost_download_path) +$(package)_file_name=$(boost_file_name) +$(package)_sha256_hash=$(boost_sha256_hash) +$(package)_build_subdir=tools/build/src/engine +ifneq (,$(findstring clang,$($(package)_cxx))) +$(package)_toolset_$(host_os)=clang +else +$(package)_toolset_$(host_os)=gcc +endif + +define $(package)_build_cmds + CXX="$($(package)_cxx)" CXXFLAGS="$($(package)_cxxflags)" ./build.sh "$($(package)_toolset_$(host_os))" +endef + +define $(package)_stage_cmds + mkdir -p "$($(package)_staging_prefix_dir)"/bin/ && \ + cp b2 "$($(package)_staging_prefix_dir)"/bin/ +endef diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk deleted file mode 100644 index c3054cbd1a..0000000000 --- a/depends/packages/native_biplist.mk +++ /dev/null @@ -1,15 +0,0 @@ -package=native_biplist -$(package)_version=1.0.3 -$(package)_download_path=https://bitbucket.org/wooster/biplist/downloads -$(package)_file_name=biplist-$($(package)_version).tar.gz -$(package)_sha256_hash=4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6 -$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages - -define $(package)_build_cmds - python3 setup.py build -endef - -define $(package)_stage_cmds - mkdir -p $($(package)_install_libdir) && \ - python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) -endef diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk deleted file mode 100644 index 7bdf2d7dfd..0000000000 --- a/depends/packages/native_cdrkit.mk +++ /dev/null @@ -1,28 +0,0 @@ -package=native_cdrkit -$(package)_version=1.1.11 -$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c -$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 -$(package)_patches=cdrkit-deterministic.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch -endef - -# Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors. -# Pass -fcommon to retain the legacy behaviour. -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -fcommon" -endef - -define $(package)_build_cmds - $(MAKE) genisoimage -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install -endef - -define $(package)_postprocess_cmds - rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump -endef diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index f99b689ecd..44108925a4 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -1,10 +1,9 @@ package=native_ds_store -$(package)_version=1.1.2 +$(package)_version=1.3.0 $(package)_download_path=https://github.com/al45tair/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=3b3ecb7bf0a5157f5b6010bc3af7c141fb0ad3527084e63336220d22744bc20c +$(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages -$(package)_dependencies=native_biplist define $(package)_build_cmds python3 setup.py build diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index e60b99dccc..5fe027fb8a 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,8 +1,8 @@ package=native_mac_alias -$(package)_version=2.0.7 +$(package)_version=2.1.1 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=6f606d3b6bccd2112aeabf1a063f5b5ece87005a5d7e97c8faca23b916e88838 +$(package)_sha256_hash=c0ffceee14f7d04a6eb323fb7b8217dc3f373b346198d2ca42300a8362db7efa $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 4627acb521..0f35ca0d2d 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -16,12 +16,15 @@ sqlite_packages=sqlite zmq_packages=zeromq upnp_packages=miniupnpc +natpmp_packages=libnatpmp multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp -darwin_native_packages = native_biplist native_ds_store native_mac_alias +darwin_native_packages = native_ds_store native_mac_alias + +$(host_arch)_$(host_os)_native_packages += native_b2 ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libdmg-hfsplus endif diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 083bc68d66..7d122ea5f9 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -6,14 +6,13 @@ $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=9b9dec1f67df1f94bce2955c5604de992d529dde72050239154c56352da0907d $(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb -$(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch $(package)_patches+= fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch $(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch $(package)_patches+= freetype_back_compat.patch drop_lrelease_dependency.patch fix_powerpc_libpng.patch +$(package)_patches+= fix_mingw_cross_compile.patch fix_qpainter_non_determinism.patch -# Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c @@ -25,9 +24,10 @@ $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars $(package)_config_opts_release = -release +$(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++11 +$(package)_config_opts += -c++std c++1z $(package)_config_opts += -confirm-license $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples @@ -68,7 +68,6 @@ $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -opensource $(package)_config_opts += -optimized-tools -$(package)_config_opts += -pch $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) $(package)_config_opts += -qt-libpng @@ -76,7 +75,6 @@ $(package)_config_opts += -qt-pcre $(package)_config_opts += -qt-harfbuzz $(package)_config_opts += -system-zlib $(package)_config_opts += -static -$(package)_config_opts += -silent $(package)_config_opts += -v $(package)_config_opts += -no-feature-bearermanagement $(package)_config_opts += -no-feature-colordialog @@ -116,6 +114,7 @@ $(package)_config_opts += -no-feature-xml $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl +$(package)_config_opts_darwin += -pch ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux @@ -124,8 +123,12 @@ $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSIO $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) +$(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif +# for macOS on Apple Silicon (ARM) see https://bugreports.qt.io/browse/QTBUG-85279 +$(package)_config_opts_arm_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 + $(package)_config_opts_linux = -qt-xkbcommon-x11 $(package)_config_opts_linux += -qt-xcb $(package)_config_opts_linux += -no-xcb-xlib @@ -147,6 +150,7 @@ $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_mingw32 += -pch $(package)_config_opts_android = -xplatform android-clang $(package)_config_opts_android += -android-sdk $(ANDROID_SDK) @@ -162,6 +166,7 @@ $(package)_config_opts_android += -qt-freetype $(package)_config_opts_android += -no-fontconfig $(package)_config_opts_android += -L $(host_prefix)/lib $(package)_config_opts_android += -I $(host_prefix)/include +$(package)_config_opts_android += -pch $(package)_config_opts_aarch64_android += -android-arch arm64-v8a $(package)_config_opts_armv7a_android += -android-arch armeabi-v7a @@ -192,33 +197,53 @@ define $(package)_extract_cmds tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef +# Preprocessing steps work as follows: +# +# 1. Apply our patches to the extracted source. See each patch for more info. +# +# 2. Point to lrelease in qttools/bin/lrelease; otherwise Qt will look for it in +# $(host)/native/bin/lrelease and not find it. +# +# 3. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. +# +# 4. After making a copy of the mkspec for the linux-arm-gnueabi host, named +# bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This +# way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't +# ship a mkspec for. See it's usage in config_opts_* above. +# +# 5. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. +# +# 6. Do similar for the win32-g++ mkspec. +# +# 7. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. +# +# 8. Adjust a regex in toolchain.prf, to accommodate Guix's usage of +# CROSS_LIBRARY_PATH. See #15277. define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/freetype_back_compat.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_powerpc_libpng.patch && \ - sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \ - patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch &&\ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_configure_mac.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch && \ + patch -p1 -i $($(package)_patch_dir)/xkb-default.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_riscv64_arch.patch && \ + patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_mingw_cross_compile.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_qpainter_non_determinism.patch &&\ + sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ - cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ - cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ - patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_configure_mac.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ - patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch &&\ - patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ - patch -p1 -i $($(package)_patch_dir)/fix_riscv64_arch.patch &&\ - patch -p1 -i $($(package)_patch_dir)/no-xlib.patch &&\ - echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ - echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf &&\ sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ @@ -231,31 +256,30 @@ define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ + cd qtbase && \ ./configure $($(package)_config_opts) && \ echo "host_build: QT_CONFIG ~= s/system-zlib/zlib" >> mkspecs/qconfig.pri && \ echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \ - $(MAKE) sub-src-clean && \ - cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \ - cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. && \ - cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile && \ - cd ../lupdate/ && ../../../../qtbase/bin/qmake lupdate.pro -o Makefile && cd ../../../.. + cd .. && \ + $(MAKE) -C qtbase sub-src-clean && \ + qtbase/bin/qmake -o qttranslations/Makefile qttranslations/qttranslations.pro && \ + qtbase/bin/qmake -o qttranslations/translations/Makefile qttranslations/translations/translations.pro && \ + qtbase/bin/qmake -o qttools/src/linguist/lrelease/Makefile qttools/src/linguist/lrelease/lrelease.pro && \ + qtbase/bin/qmake -o qttools/src/linguist/lupdate/Makefile qttools/src/linguist/lupdate/lupdate.pro endef define $(package)_build_cmds - $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \ - $(MAKE) -C ../qttools/src/linguist/lrelease && \ - $(MAKE) -C ../qttools/src/linguist/lupdate && \ - $(MAKE) -C ../qttranslations + $(MAKE) -C qtbase/src $(addprefix sub-,$($(package)_qt_libs)) && \ + $(MAKE) -C qttools/src/linguist/lrelease && \ + $(MAKE) -C qttools/src/linguist/lupdate && \ + $(MAKE) -C qttranslations endef define $(package)_stage_cmds - $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. && \ + $(MAKE) -C qtbase/src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && \ $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ $(MAKE) -C qttools/src/linguist/lupdate INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets && \ - if `test -f qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a`; then \ - cp qtbase/src/plugins/platforms/xcb/xcb-static/libxcb-static.a $($(package)_staging_prefix_dir)/lib; \ - fi + $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets endef define $(package)_postprocess_cmds diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index c93aa1a74d..3b7f3690a4 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -12,7 +12,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic - $(package)_cxxflags=-std=c++11 + $(package)_cxxflags=-std=c++17 endef define $(package)_preprocess_cmds diff --git a/depends/patches/boost/unused_var_in_process.patch b/depends/patches/boost/unused_var_in_process.patch deleted file mode 100644 index 722f7bb5ea..0000000000 --- a/depends/patches/boost/unused_var_in_process.patch +++ /dev/null @@ -1,22 +0,0 @@ -commit dbd95cdaefdea95307d004f019a1c394cf9389f0 -Author: fanquake <fanquake@gmail.com> -Date: Mon Aug 17 20:15:17 2020 +0800 - - Remove unused variable in Boost Process - - This causes issues with our linters / CI. - - Can be removed once depends Boost is 1.71.0 or later. - -diff --git a/boost/process/detail/posix/wait_group.hpp b/boost/process/detail/posix/wait_group.hpp -index 9dc249803..2502d9772 100644 ---- a/boost/process/detail/posix/wait_group.hpp -+++ b/boost/process/detail/posix/wait_group.hpp -@@ -137,7 +137,6 @@ inline bool wait_until( - - do - { -- int ret_sig = 0; - int status; - if ((::waitpid(timeout_pid, &status, WNOHANG) != 0) - && (WIFEXITED(status) || WIFSIGNALED(status))) diff --git a/depends/patches/fontconfig/gperf_header_regen.patch b/depends/patches/fontconfig/gperf_header_regen.patch index 7401b83d84..3ffd1674e0 100644 --- a/depends/patches/fontconfig/gperf_header_regen.patch +++ b/depends/patches/fontconfig/gperf_header_regen.patch @@ -2,7 +2,7 @@ commit 7b6eb33ecd88768b28c67ce5d2d68a7eed5936b6 Author: fanquake <fanquake@gmail.com> Date: Tue Aug 25 14:34:53 2020 +0800 - Remove rule that causes inadvertant header regeneration + Remove rule that causes inadvertent header regeneration Otherwise the makefile will needlessly attempt to re-generate the headers with gperf. This can be dropped once the upstream build is fixed. diff --git a/depends/patches/native_cctools/ld64_disable_threading.patch b/depends/patches/native_cctools/ld64_disable_threading.patch index d6c58c102f..2de6874cd4 100644 --- a/depends/patches/native_cctools/ld64_disable_threading.patch +++ b/depends/patches/native_cctools/ld64_disable_threading.patch @@ -8,7 +8,7 @@ Date: Tue Aug 18 01:20:24 2020 +0000 differently based on which files have already been parsed. This is more likely to occur on systems with more CPUs. - Just disable threading for now. There is no noticable slowdown. + Just disable threading for now. There is no noticeable slowdown. See #9891. diff --git a/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/depends/patches/native_cdrkit/cdrkit-deterministic.patch deleted file mode 100644 index 8ab0993dc4..0000000000 --- a/depends/patches/native_cdrkit/cdrkit-deterministic.patch +++ /dev/null @@ -1,86 +0,0 @@ ---- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 -+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 -@@ -1139,8 +1139,9 @@ - scan_directory_tree(struct directory *this_dir, char *path, - struct directory_entry *de) - { -- DIR *current_dir; -+ int current_file; - char whole_path[PATH_MAX]; -+ struct dirent **d_list; - struct dirent *d_entry; - struct directory *parent; - int dflag; -@@ -1164,7 +1165,8 @@ - this_dir->dir_flags |= DIR_WAS_SCANNED; - - errno = 0; /* Paranoia */ -- current_dir = opendir(path); -+ //current_dir = opendir(path); -+ current_file = scandir(path, &d_list, NULL, alphasort); - d_entry = NULL; - - /* -@@ -1173,12 +1175,12 @@ - */ - old_path = path; - -- if (current_dir) { -+ if (current_file >= 0) { - errno = 0; -- d_entry = readdir(current_dir); -+ d_entry = d_list[0]; - } - -- if (!current_dir || !d_entry) { -+ if (current_file < 0 || !d_entry) { - int ret = 1; - - #ifdef USE_LIBSCHILY -@@ -1191,8 +1193,8 @@ - de->isorec.flags[0] &= ~ISO_DIRECTORY; - ret = 0; - } -- if (current_dir) -- closedir(current_dir); -+ if(d_list) -+ free(d_list); - return (ret); - } - #ifdef ABORT_DEEP_ISO_ONLY -@@ -1208,7 +1210,7 @@ - errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); - errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); - } -- closedir(current_dir); -+ free(d_list); - return (1); - } - #endif -@@ -1250,13 +1252,13 @@ - * The first time through, skip this, since we already asked - * for the first entry when we opened the directory. - */ -- if (dflag) -- d_entry = readdir(current_dir); -+ if (dflag && current_file >= 0) -+ d_entry = d_list[current_file]; - dflag++; - -- if (!d_entry) -+ if (current_file < 0) - break; -- -+ current_file--; - /* OK, got a valid entry */ - - /* If we do not want all files, then pitch the backups. */ -@@ -1348,7 +1350,7 @@ - insert_file_entry(this_dir, whole_path, d_entry->d_name); - #endif /* APPLE_HYB */ - } -- closedir(current_dir); -+ free(d_list); - - #ifdef APPLE_HYB - /* diff --git a/depends/patches/qt/fix_mingw_cross_compile.patch b/depends/patches/qt/fix_mingw_cross_compile.patch new file mode 100644 index 0000000000..67f76f1d85 --- /dev/null +++ b/depends/patches/qt/fix_mingw_cross_compile.patch @@ -0,0 +1,25 @@ +commit 5a992a549adfe5a587bbcd6cd2b2cee47d236e27 +Author: fanquake <fanquake@gmail.com> +Date: Fri Sep 4 08:13:44 2020 +0800 + + Work around broken mingw cross-compilation + + See upstream issues: + https://bugreports.qt.io/browse/QTBUG-63637 + https://bugreports.qt.io/browse/QTBUG-63659 + https://codereview.qt-project.org/q/8bebded9 + + We should be able to drop this once we are building qt 5.10.1 or later. + + Added in #12971. + +diff --git a/qtbase/mkspecs/win32-g++/qmake.conf b/qtbase/mkspecs/win32-g++/qmake.conf +index e071a0d1..ad229b10 100644 +--- a/qtbase/mkspecs/win32-g++/qmake.conf ++++ b/qtbase/mkspecs/win32-g++/qmake.conf +@@ -87,3 +87,5 @@ QMAKE_NM = $${CROSS_COMPILE}nm -P + include(../common/angle.conf) + + load(qt_config) ++QMAKE_LINK_OBJECT_MAX = 10 ++QMAKE_LINK_OBJECT_SCRIPT = object_script diff --git a/depends/patches/qt/fix_qpainter_non_determinism.patch b/depends/patches/qt/fix_qpainter_non_determinism.patch new file mode 100644 index 0000000000..3cfcc22f03 --- /dev/null +++ b/depends/patches/qt/fix_qpainter_non_determinism.patch @@ -0,0 +1,63 @@ +commit 2a8f7dc6ddfc414a66491522501c1574a1343ee1 +Author: Andrew Chow <achow101-github@achow101.com> +Date: Sat Nov 21 01:11:04 2020 -0500 + + build: Fix determinism issue when building with Clang 8 + + When building Qt with LLVM/Clang 8 under -O3 (the default), we run into + a determinism issue in `qt_interset_spans`. The issue has been fixed for + LLVM/Clang 9, see + https://github.com/llvm/llvm-project/commit/db101864bdc938deb1d63fe4f7da761bd38e5cae + and https://reviews.llvm.org/D64601, however this fix was not backported + to 8.x. Once LLVM/Clang 9 is used, this patch can be dropped. + + The particular issue appears to be an optimization done by -O3 which + adds a temporary variable for `spans->y` in `qt_intersect_spans`. When + it does this, sometimes it chooses to use a 32-bit movs instruction + (movswl), and other times it chooses a 64-bit movs instruction (movswq). + By patching `qt_intersect_spans` to always make a temporary variable for + `spans->y`, we are able to sidestep this problem. + +diff --git a/qtbase/src/gui/painting/qpaintengine_raster.cpp b/qtbase/src/gui/painting/qpaintengine_raster.cpp +index 92ab6e8375..f018009e0b 100644 +--- a/qtbase/src/gui/painting/qpaintengine_raster.cpp ++++ b/qtbase/src/gui/painting/qpaintengine_raster.cpp +@@ -3971,22 +3971,23 @@ static const QSpan *qt_intersect_spans(const QClipData *clip, int *currentClip, + const QSpan *clipEnd = clip->m_spans + clip->count; + + while (available && spans < end ) { ++ const short spans_y = spans->y; + if (clipSpans >= clipEnd) { + spans = end; + break; + } +- if (clipSpans->y > spans->y) { ++ if (clipSpans->y > spans_y) { + ++spans; + continue; + } +- if (spans->y != clipSpans->y) { +- if (spans->y < clip->count && clip->m_clipLines[spans->y].spans) +- clipSpans = clip->m_clipLines[spans->y].spans; ++ if (spans_y != clipSpans->y) { ++ if (spans_y < clip->count && clip->m_clipLines[spans_y].spans) ++ clipSpans = clip->m_clipLines[spans_y].spans; + else + ++clipSpans; + continue; + } +- Q_ASSERT(spans->y == clipSpans->y); ++ Q_ASSERT(spans_y == clipSpans->y); + + int sx1 = spans->x; + int sx2 = sx1 + spans->len; +@@ -4005,7 +4006,7 @@ static const QSpan *qt_intersect_spans(const QClipData *clip, int *currentClip, + if (len) { + out->x = qMax(sx1, cx1); + out->len = qMin(sx2, cx2) - out->x; +- out->y = spans->y; ++ out->y = spans_y; + out->coverage = qt_div_255(spans->coverage * clipSpans->coverage); + ++out; + --available; + diff --git a/depends/patches/qt/freetype_back_compat.patch b/depends/patches/qt/freetype_back_compat.patch index b0f1c98aa6..1ca55f1ce3 100644 --- a/depends/patches/qt/freetype_back_compat.patch +++ b/depends/patches/qt/freetype_back_compat.patch @@ -9,7 +9,7 @@ Date: Tue Aug 18 15:15:08 2020 +0800 backwards-compatibility. Qt 5.9 introduced a call to FT_Get_Font_Format(). Replace it with FT_Get_X11_Font_Format() - in order to remain compatibile with older freetype, which is still used by e.g. Ubuntu Trusty. + in order to remain compatible with older freetype, which is still used by e.g. Ubuntu Trusty. See #14348. diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 4cd96df29f..0142667547 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -1,12 +1,12 @@ MAKEFILE_GENERATOR = UNIX -CONFIG += app_bundle incremental global_init_link_order lib_version_first plugin_no_soname absolute_library_soname +CONFIG += app_bundle incremental lib_version_first absolute_library_soname QMAKE_INCREMENTAL_STYLE = sublib include(../common/macx.conf) include(../common/gcc-base-mac.conf) include(../common/clang.conf) include(../common/clang-mac.conf) QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} -QMAKE_XCODE_VERSION=4.3 +QMAKE_XCODE_VERSION = $${XCODE_VERSION} QMAKE_XCODE_DEVELOPER_PATH=/Developer QMAKE_MACOSX_DEPLOYMENT_TARGET = $${MAC_MIN_VERSION} QMAKE_MAC_SDK=macosx diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 40d8e330e2..12807bfb86 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -88,13 +88,14 @@ RPC interface will be abused. - **Secure string handling:** The RPC interface does not guarantee any escaping of data beyond what's necessary to encode it as JSON, although it does usually provide serialized data using a hex - representation of the bytes. If you use RPC data in your programs or - provide its data to other programs, you must ensure any problem - strings are properly escaped. For example, multiple websites have - been manipulated because they displayed decoded hex strings that - included HTML `<script>` tags. For this reason, and other - non-security reasons, it is recommended to display all serialized data - in hex form only. + representation of the bytes. If you use RPC data in your programs or + provide its data to other programs, you must ensure any problem strings + are properly escaped. For example, the `createwallet` RPC accepts + arguments such as `wallet_name` which is a string and could be used + for a path traversal attack without application level checks. Multiple + websites have been manipulated because they displayed decoded hex strings + that included HTML `<script>` tags. For this reason, and others, it is + recommended to display all serialized data in hex form only. ## RPC consistency guarantees @@ -127,3 +128,14 @@ However, the wallet may not be up-to-date with the current state of the mempool or the state of the mempool by an RPC that returned before this RPC. For example, a wallet transaction that was BIP-125-replaced in the mempool prior to this RPC may not yet be reflected as such in this RPC response. + +## Limitations + +There is a known issue in the JSON-RPC interface that can cause a node to crash if +too many http connections are being opened at the same time because the system runs +out of available file descriptors. To prevent this from happening you might +want to increase the number of maximum allowed file descriptors in your system +and try to prevent opening too many connections to your JSON-RPC interface at the +same time if this is under your control. It is hard to give general advice +since this depends on your system but if you make several hundred requests at +once you are definitely at risk of encountering this issue. diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 842a3964df..6237734390 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -4,7 +4,7 @@ Unauthenticated REST Interface The REST API can be enabled with the `-rest` option. The interface runs on the same port as the JSON-RPC interface, by default port 8332 for mainnet, port 18332 for testnet, -and port 18443 for regtest. +port 38332 for signet, and port 18443 for regtest. REST Interface consistency guarantees ------------------------------------- @@ -12,6 +12,18 @@ REST Interface consistency guarantees The [same guarantees as for the RPC Interface](/doc/JSON-RPC-interface.md#rpc-consistency-guarantees) apply. +Limitations +----------- + +There is a known issue in the REST interface that can cause a node to crash if +too many http connections are being opened at the same time because the system runs +out of available file descriptors. To prevent this from happening you might +want to increase the number of maximum allowed file descriptors in your system +and try to prevent opening too many connections to your rest interface at the +same time if this is under your control. It is hard to give general advice +since this depends on your system but if you make several hundred requests at +once you are definitely at risk of encountering this issue. + Supported API ------------- @@ -50,7 +62,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name (main, test, regtest) +* chain : (string) current network name (main, test, signet, regtest) * blocks : (numeric) the current number of blocks processed in the server * headers : (numeric) the current number of headers we have validated * bestblockhash : (string) the hash of the currently best block diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index f48855a344..18ea84c579 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,59 +1,127 @@ -FreeBSD build guide -====================== -(updated for FreeBSD 12.0) +# FreeBSD Build Guide -This guide describes how to build bitcoind and command-line utilities on FreeBSD. +**Updated for FreeBSD [12.2](https://www.freebsd.org/releases/12.2R/announce.html)** -This guide does not contain instructions for building the GUI. +This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. + +## Dependencies + +The following dependencies are **required**: + + Library | Purpose | Description + ----------------------------------------------------------------------|------------|---------------------- + [autoconf](https://svnweb.freebsd.org/ports/head/devel/autoconf/) | Build | Automatically configure software source code + [automake](https://svnweb.freebsd.org/ports/head/devel/automake/) | Build | Generate makefile (requires autoconf) + [libtool](https://svnweb.freebsd.org/ports/head/devel/libtool/) | Build | Shared library support + [pkgconf](https://svnweb.freebsd.org/ports/head/devel/pkgconf/) | Build | Configure compiler and linker flags + [git](https://svnweb.freebsd.org/ports/head/devel/git/) | Clone | Version control system + [gmake](https://svnweb.freebsd.org/ports/head/devel/gmake/) | Compile | Generate executables + [boost-libs](https://svnweb.freebsd.org/ports/head/devel/boost-libs/) | Utility | Library for threading, data structures, etc + [libevent](https://svnweb.freebsd.org/ports/head/devel/libevent/) | Networking | OS independent asynchronous networking + + +The following dependencies are **optional**: + + Library | Purpose | Description + ---------------------------------------------------------------------------|------------------|---------------------- + [db5](https://svnweb.freebsd.org/ports/head/databases/db5/) | Berkeley DB | Wallet storage (only needed when wallet enabled) + [qt5](https://svnweb.freebsd.org/ports/head/devel/qt5/) | GUI | GUI toolkit (only needed when GUI enabled) + [libqrencode](https://svnweb.freebsd.org/ports/head/graphics/libqrencode/) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) + [libzmq4](https://svnweb.freebsd.org/ports/head/net/libzmq4/) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) + [sqlite3](https://svnweb.freebsd.org/ports/head/databases/sqlite3/) | SQLite DB | Wallet storage (only needed when wallet enabled) + [python3](https://svnweb.freebsd.org/ports/head/lang/python3/) | Testing | Python Interpreter (only needed when running the test suite) + + See [dependencies.md](dependencies.md) for a complete overview. ## Preparation -You will need the following dependencies, which can be installed as root via pkg: +### 1. Install Required Dependencies +Install the required dependencies the usual way you [install software on FreeBSD](https://www.freebsd.org/doc/en/books/handbook/ports.html) - either with `pkg` or via the Ports collection. The example commands below use `pkg` which is usually run as `root` or via `sudo`. If you want to use `sudo`, and you haven't set it up: [use this guide](http://www.freebsdwiki.net/index.php/Sudo%2C_configuring) to setup `sudo` access on FreeBSD. ```bash pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf +``` + +### 2. Clone Bitcoin Repo +Now that `git` and all the required dependencies are installed, let's clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. +``` bash git clone https://github.com/bitcoin/bitcoin.git ``` -In order to run the test suite (recommended), you will need to have Python 3 installed: +### 3. Install Optional Dependencies + +#### Wallet Dependencies +It is not necessary to build wallet functionality to run bitcoind or the GUI. To enable legacy wallets, you must install `db5`. To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite3` is required. Skip `db5` if you intend to *exclusively* use descriptor wallets + +###### Legacy Wallet Support +`db5` is required to enable support for legacy wallets. Skip if you don't intend to use legacy wallets ```bash -pkg install python3 +pkg install db5 ``` -See [dependencies.md](dependencies.md) for a complete overview. +###### Descriptor Wallet Support -### Building BerkeleyDB +`sqlite3` is required to enable support for descriptor wallets. Skip if you don't intend to use descriptor wallets. +``` bash +pkg install sqlite3 +``` +--- -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +#### GUI Dependencies +###### Qt5 +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. Skip if you don't intend to use the GUI. ```bash -./contrib/install_db4.sh `pwd` -export BDB_PREFIX="$PWD/db4" +pkg install qt5 ``` +###### libqrencode + +The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `libqrencode`. Skip if not using the GUI or don't want QR code functionality. +```bash +pkg install libqrencode +``` +--- + +#### Test Suite Dependencies +There is an included test suite that is useful for testing code changes when developing. +To run the test suite (recommended), you will need to have Python 3 installed: + +```bash +pkg install python3 +``` +--- ## Building Bitcoin Core -**Important**: Use `gmake` (the non-GNU `make` will exit with an error). +### 1. Configuration -With wallet: +There are many ways to configure Bitcoin Core, here are a few common examples: +##### Wallet (BDB + SQlite) Support, No GUI: +This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` is installed, then descriptor wallet support will be built. ```bash ./autogen.sh -./configure --with-gui=no \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ +./configure --with-gui=no --with-incompatible-bdb \ + BDB_LIBS="-ldb_cxx-5" \ + BDB_CFLAGS="-I/usr/local/include/db5" \ MAKE=gmake ``` -Without wallet: +##### Wallet (only SQlite) and GUI Support: +This explicitly enables the GUI and disables legacy wallet support. If `qt5` is not installed, this will throw an error. If `sqlite3` is installed then descriptor wallet functionality will be built. If `sqlite3` is not installed, then wallet functionality will be disabled. ```bash ./autogen.sh -./configure --with-gui=no --disable-wallet MAKE=gmake +./configure --without-bdb --with-gui=yes MAKE=gmake +``` +##### No Wallet or GUI +``` bash +./autogen.sh +./configure --without-wallet --with-gui=no MAKE=gmake ``` -followed by: +### 2. Compile +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). ```bash gmake # use -jX here for parallelism diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 2b051c078c..dccd7b1335 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -15,6 +15,7 @@ pkg_add qt5 # (optional for enabling the GUI) pkg_add autoconf # (select highest version, e.g. 2.69) pkg_add automake # (select highest version, e.g. 1.16) pkg_add python # (select highest version, e.g. 3.8) +pkg_add bash git clone https://github.com/bitcoin/bitcoin.git ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 2a7d71eea6..04ee43f81d 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -19,7 +19,7 @@ Then install [Homebrew](https://brew.sh). ## Dependencies ```shell -brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode sqlite +brew install automake libtool boost miniupnpc libnatpmp pkg-config python qt libevent qrencode ``` If you run into issues, check [Homebrew's troubleshooting page](https://docs.brew.sh/Troubleshooting). @@ -30,7 +30,27 @@ If you want to build the disk image with `make deploy` (.dmg / optional), you ne brew install librsvg ``` -## Berkeley DB +and [`macdeployqtplus`](../contrib/macdeploy/README.md) dependencies: +```shell +pip3 install ds_store mac_alias +``` + +The wallet support requires one or both of the dependencies ([*SQLite*](#sqlite) and [*Berkeley DB*](#berkeley-db)) in the sections below. +To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode). + +#### SQLite + +Usually, macOS installation already has a suitable SQLite installation. +Also, the Homebrew package could be installed: + +```shell +brew install sqlite +``` + +In that case the Homebrew package will prevail. + +#### Berkeley DB + It is recommended to use Berkeley DB 4.8. If you have to build it yourself, you can use [this](/contrib/install_db4.sh) script to install it like so: @@ -41,7 +61,11 @@ like so: from the root of the repository. -**Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](/doc/build-osx.md#disable-wallet-mode)). +Also, the Homebrew package could be installed: + +```shell +brew install berkeley-db4 +``` ## Build Bitcoin Core @@ -72,14 +96,14 @@ from the root of the repository. make deploy ``` -## `disable-wallet` mode +## Disable-wallet mode When the intention is to run only a P2P node without a wallet, Bitcoin Core may be -compiled in `disable-wallet` mode with: +compiled in disable-wallet mode with: ```shell ./configure --disable-wallet ``` -In this case there is no dependency on Berkeley DB 4.8 and SQLite. +In this case there is no dependency on [*Berkeley DB*](#berkeley-db) and [*SQLite*](#sqlite). Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. @@ -111,6 +135,6 @@ tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log ``` ## Notes -* Tested on OS X 10.12 Sierra through macOS 10.15 Catalina on 64-bit Intel +* Tested on OS X 10.14 Mojave through macOS 11 Big Sur on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). diff --git a/doc/build-unix.md b/doc/build-unix.md index ab83ceff3d..5c24886dbf 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -41,7 +41,8 @@ Optional dependencies: Library | Purpose | Description ------------|------------------|---------------------- miniupnpc | UPnP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Wallet storage (only needed when wallet enabled) + libnatpmp | NAT-PMP Support | Firewall-jumping support + libdb4.8 | Berkeley DB | Optional, wallet storage (only needed when wallet enabled) qt | GUI | GUI toolkit (only needed when GUI enabled) libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) @@ -99,9 +100,9 @@ SQLite is required for the wallet: To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) -Optional (see `--with-miniupnpc` and `--enable-upnp-default`): +Optional port mapping libraries (see: `--with-miniupnpc`, and `--enable-upnp-default`, `--with-natpmp`, `--enable-natpmp-default`): - sudo apt-get install libminiupnpc-dev + sudo apt install libminiupnpc-dev libnatpmp-dev ZMQ dependencies (provides ZMQ API): @@ -133,9 +134,9 @@ Build requirements: sudo dnf install gcc-c++ libtool make autoconf automake libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 -Optional (see `--with-miniupnpc` and `--enable-upnp-default`): +Optional port mapping libraries (see: `--with-miniupnpc`, and `--enable-upnp-default`, `--with-natpmp`, `--enable-natpmp-default`): - sudo dnf install miniupnpc-devel + sudo dnf install miniupnpc-devel libnatpmp-devel ZMQ dependencies (provides ZMQ API): @@ -158,18 +159,27 @@ Notes The release is built with GCC and then "strip bitcoind" to strip the debug symbols, which reduces the executable size by about 90%. - miniupnpc --------- [miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. See the configure options for upnp behavior desired: +turned off by default. See the configure options for UPnP behavior desired: - --without-miniupnpc No UPnP support miniupnp not required + --without-miniupnpc No UPnP support, miniupnp not required --disable-upnp-default (the default) UPnP support turned off by default at runtime --enable-upnp-default UPnP support turned on by default at runtime +libnatpmp +--------- + +[libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded +from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and +turned off by default. See the configure options for NAT-PMP behavior desired: + + --without-natpmp No NAT-PMP support, libnatpmp not required + --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime + --enable-natpmp-default NAT-PMP support turned on by default at runtime Berkeley DB ----------- diff --git a/doc/dependencies.md b/doc/dependencies.md index d1bf4b3a87..6b2b904632 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -6,19 +6,20 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | | --- | --- | --- | --- | --- | --- | | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.70.0](https://www.boost.org/users/download/) | [1.58.0](https://github.com/bitcoin/bitcoin/pull/19667) | No | | | -| Clang | | [3.3+](https://releases.llvm.org/download.html) (C++11 support) | | | | +| Boost | [1.71.0](https://www.boost.org/users/download/) | [1.58.0](https://github.com/bitcoin/bitcoin/pull/19667) | No | | | +| Clang | | [5.0+](https://releases.llvm.org/download.html) (C++17 support) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | +| GCC | | [7+](https://gcc.gnu.org/) (C++17 support) | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | +| libnatpmp | [20150609](https://miniupnp.tuxfamily.org/files) | | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](https://miniupnp.tuxfamily.org/files) | | No | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| Python (tests) | | [3.5](https://www.python.org/downloads) | | | | +| Python (tests) | | [3.6](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | @@ -32,8 +33,9 @@ Controlling dependencies Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. #### Options passed to `./configure` -* MiniUPnPc is not needed with `--with-miniupnpc=no`. -* Berkeley DB is not needed with `--disable-wallet`. +* MiniUPnPc is not needed with `--without-miniupnpc`. +* libnatpmp is not needed with `--without-natpmp`. +* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`. * SQLite is not needed with `--disable-wallet` or `--without-sqlite`. * Qt is not needed with `--without-gui`. * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. diff --git a/doc/descriptors.md b/doc/descriptors.md index 181ff77e50..63acb9167f 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -50,7 +50,7 @@ Output descriptors currently support: - `wsh(multi(2,03a0434d9e47f3c86235477c7b1ae6ae5d3442d49b1943c2b752a68e2a47e247c7,03774ae7f858a9411e5ef4246b70c65aac5649980be5c17891bbec17895da008cb,03d01115d548e7561b15c38f004d734633687cf4419620095bc5b0f47070afe85a))` describes a P2WSH *2-of-3* multisig output with keys in the specified order. - `sh(wsh(multi(1,03f28773c2d975288bc7d1d205c3748651b075fbc6610e58cddeeddf8f19405aa8,03499fdf9e895e719cfd64e67f07d38e3226aa7b63678949e6e49b241a60e823e4,02d7924d4f7d43ea965a465ae3095ff41131e5946f3c85f79e44adbcf8e27e080e)))` describes a P2SH-P2WSH *1-of-3* multisig output with keys in the specified order. - `pk(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8)` describes a P2PK output with the public key of the specified xpub. -- `pkh(xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw/1'/2)` describes a P2PKH output with child key *1'/2* of the specified xpub. +- `pkh(xpub68Gmy5EdvgibQVfPdqkBBCHxA5htiqg55crXYuXoQRKfDBFA1WEjWgP6LHhwBZeNK1VTsfTFUHCdrfp1bgwQ9xv5ski8PX9rL2dZXvgGDnw/1/2)` describes a P2PKH output with child key *1/2* of the specified xpub. - `pkh([d34db33f/44'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*)` describes a set of P2PKH outputs, but additionally specifies that the specified xpub is a child of a master with fingerprint `d34db33f`, and derived using path `44'/0'/0'`. - `wsh(multi(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where the first multisig key is the *1/0/`i`* child of the first specified xpub and the second multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). - `wsh(sortedmulti(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where one multisig key is the *1/0/`i`* child of the first specified xpub and the other multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). The order of public keys in the resulting witnessScripts is determined by the lexicographic order of the public keys at that index. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index fa188dbcd6..596f65cf10 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -14,7 +14,7 @@ Developer Notes - [Compiling for debugging](#compiling-for-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - - [Testnet and Regtest modes](#testnet-and-regtest-modes) + - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) @@ -135,7 +135,7 @@ Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.m Coding Style (Doxygen-compatible comments) ------------------------------------------ -Bitcoin Core uses [Doxygen](http://www.doxygen.nl/) to generate its official documentation. +Bitcoin Core uses [Doxygen](https://www.doxygen.nl/) to generate its official documentation. Use Doxygen-compatible comment blocks for functions, methods, and fields. @@ -156,7 +156,7 @@ For example, to describe a function use: bool function(int arg1, const char *arg2, std::string& arg3) ``` -A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. +A complete list of `@xxx` commands can be found at https://www.doxygen.nl/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. @@ -203,7 +203,7 @@ Also not picked up by Doxygen: */ ``` -A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, +A full list of comment syntaxes picked up by Doxygen can be found at https://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. Recommendations: @@ -216,7 +216,7 @@ Recommendations: - Backticks aren't required when referring to functions Doxygen already knows about; it will build hyperlinks for these automatically. See - http://www.doxygen.nl/manual/autolink.html for complete info. + https://www.doxygen.nl/manual/autolink.html for complete info. - Avoid linking to external documentation; links can break. @@ -259,14 +259,15 @@ on all categories (and give you a very large `debug.log` file). The Qt code routes `qDebug()` output to `debug.log` under category "qt": run with `-debug=qt` to see it. -### Testnet and Regtest modes +### Signet, testnet, and regtest modes -Run with the `-testnet` option to run with "play bitcoins" on the test network, if you -are testing multi-machine code that needs to operate across the internet. +If you are testing multi-machine code that needs to operate across the internet, +you can run with either the `-signet` or the `-testnet` config option to test +with "play bitcoins" on a test network. -If you are testing something that can run on one machine, run with the `-regtest` option. -In regression test mode, blocks can be created on-demand; see [test/functional/](/test/functional) for tests -that run in `-regtest` mode. +If you are testing something that can run on one machine, run with the +`-regtest` option. In regression test mode, blocks can be created on demand; +see [test/functional/](/test/functional) for tests that run in `-regtest` mode. ### DEBUG_LOCKORDER @@ -276,6 +277,33 @@ configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the `debug.log` file if inconsistencies are detected. +### Assertions and Checks + +The util file `src/util/check.h` offers helpers to protect against coding and +internal logic bugs. They must never be used to validate user, network or any +other input. + +* `assert` or `Assert` should be used to document assumptions when any + violation would mean that it is not safe to continue program execution. The + code is always compiled with assertions enabled. + - For example, a nullptr dereference or any other logic bug in validation + code means the program code is faulty and must terminate immediately. +* `CHECK_NONFATAL` should be used for recoverable internal logic bugs. On + failure, it will throw an exception, which can be caught to recover from the + error. + - For example, a nullptr dereference or any other logic bug in RPC code + means that the RPC code is faulty and can not be executed. However, the + logic bug can be shown to the user and the program can continue to run. +* `Assume` should be used to document assumptions when program execution can + safely continue even if the assumption is violated. In debug builds it + behaves like `Assert`/`assert` to notify developers and testers about + nonfatal errors. In production it doesn't warn or log anything, though the + expression is always evaluated. + - For example it can be assumed that a variable is only initialized once, + but a failed assumption does not result in a fatal bug. A failed + assumption may or may not result in a slightly degraded user experience, + but it is safe to continue program execution. + ### Valgrind suppressions file Valgrind is a programming tool for memory debugging, memory leak detection, and @@ -516,7 +544,7 @@ General Bitcoin Core - *Rationale*: RPC allows for better automatic testing. The test suite for the GUI is very limited. -- Make sure pull requests pass Travis CI before merging. +- Make sure pull requests pass CI before merging. - *Rationale*: Makes sure that they pass thorough testing, and that the tester will keep passing on the master branch. Otherwise, all new pull requests will start failing the tests, resulting in @@ -1009,7 +1037,7 @@ Scripted diffs -------------- For reformatting and refactoring commits where the changes can be easily automated using a bash script, we use -scripted-diff commits. The bash script is included in the commit message and our Travis CI job checks that +scripted-diff commits. The bash script is included in the commit message and our CI job checks that the result of the script is identical to the commit. This aids reviewers since they can verify that the script does exactly what it is supposed to do. It is also helpful for rebasing (since the same script can just be re-run on the new master commit). diff --git a/doc/fuzzing.md b/doc/fuzzing.md index c97b8d4d50..80ce821091 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -12,7 +12,7 @@ $ CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzze # macOS users: If you have problem with this step then make sure to read "macOS hints for # libFuzzer" on https://github.com/bitcoin/bitcoin/blob/master/doc/fuzzing.md#macos-hints-for-libfuzzer $ make -$ src/test/fuzz/process_message +$ FUZZ=process_message src/test/fuzz/fuzz # abort fuzzing using ctrl-c ``` @@ -26,7 +26,7 @@ If you specify a corpus directory then any new coverage increasing inputs will b ```sh $ mkdir -p process_message-seeded-from-thin-air/ -$ src/test/fuzz/process_message process_message-seeded-from-thin-air/ +$ FUZZ=process_message src/test/fuzz/fuzz process_message-seeded-from-thin-air/ INFO: Seed: 840522292 INFO: Loaded 1 modules (424174 inline 8-bit counters): 424174 [0x55e121ef9ab8, 0x55e121f613a6), INFO: Loaded 1 PC tables (424174 PCs): 424174 [0x55e121f613a8,0x55e1225da288), @@ -70,7 +70,7 @@ To fuzz `process_message` using the [`bitcoin-core/qa-assets`](https://github.co ```sh $ git clone https://github.com/bitcoin-core/qa-assets -$ src/test/fuzz/process_message qa-assets/fuzz_seed_corpus/process_message/ +$ FUZZ=process_message src/test/fuzz/fuzz qa-assets/fuzz_seed_corpus/process_message/ INFO: Seed: 1346407872 INFO: Loaded 1 modules (424174 inline 8-bit counters): 424174 [0x55d8a9004ab8, 0x55d8a906c3a6), INFO: Loaded 1 PC tables (424174 PCs): 424174 [0x55d8a906c3a8,0x55d8a96e5288), @@ -129,7 +129,7 @@ $ make # try compiling using: AFL_NO_X86=1 make $ mkdir -p inputs/ outputs/ $ echo A > inputs/thin-air-input -$ afl/afl-fuzz -i inputs/ -o outputs/ -- src/test/fuzz/bech32 +$ FUZZ=bech32 afl/afl-fuzz -i inputs/ -o outputs/ -- src/test/fuzz/fuzz # You may have to change a few kernel parameters to test optimally - afl-fuzz # will print an error and suggestion if so. ``` @@ -153,7 +153,7 @@ $ cd .. $ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined $ make $ mkdir -p inputs/ -$ honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/process_message +$ FUZZ=process_message honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/fuzz ``` Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. diff --git a/doc/guix.md b/doc/guix.md new file mode 100644 index 0000000000..fa9c74f486 --- /dev/null +++ b/doc/guix.md @@ -0,0 +1,3 @@ +# Bootstrappable Bitcoin Core Builds + +See [contrib/guix/README.md](../contrib/guix/README.md) diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index edbc0911a1..8f890da532 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -16,6 +16,10 @@ if BUILD_BITCOIN_TX dist_man1_MANS+=bitcoin-tx.1 endif +if BUILD_BITCOIN_UTIL + dist_man1_MANS+=bitcoin-util.1 +endif + if ENABLE_WALLET if BUILD_BITCOIN_WALLET dist_man1_MANS+=bitcoin-wallet.1 diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 129651d8e9..6bcad7006b 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -1,115 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-CLI "1" "February 2019" "bitcoin-cli v0.17.99.0" "User Commands" +.TH BITCOIN-CLI "1" .SH NAME -bitcoin-cli \- manual page for bitcoin-cli v0.17.99.0 -.SH SYNOPSIS -.B bitcoin-cli -[\fI\,options\/\fR] \fI\,<command> \/\fR[\fI\,params\/\fR] \fI\,Send command to Bitcoin Core\/\fR -.br -.B bitcoin-cli -[\fI\,options\/\fR] \fI\,-named <command> \/\fR[\fI\,name=value\/\fR]... \fI\,Send command to Bitcoin Core (with named arguments)\/\fR -.br -.B bitcoin-cli -[\fI\,options\/\fR] \fI\,help List commands\/\fR -.br -.B bitcoin-cli -[\fI\,options\/\fR] \fI\,help <command> Get help for a command\/\fR -.SH DESCRIPTION -Bitcoin Core RPC client version v0.17.99.0 -.SH OPTIONS -.HP -\-? -.IP -Print this help message and exit -.HP -\fB\-conf=\fR<file> -.IP -Specify configuration file. Relative paths will be prefixed by datadir -location. (default: bitcoin.conf) -.HP -\fB\-datadir=\fR<dir> -.IP -Specify data directory -.HP -\fB\-getinfo\fR -.IP -Get general information from the remote server. Note that unlike -server\-side RPC calls, the results of \fB\-getinfo\fR is the result of -multiple non\-atomic requests. Some entries in the result may -represent results from different states (e.g. wallet balance may -be as of a different block from the chain state reported) -.HP -\fB\-named\fR -.IP -Pass named instead of positional arguments (default: false) -.HP -\fB\-rpcclienttimeout=\fR<n> -.IP -Timeout in seconds during HTTP requests, or 0 for no timeout. (default: -900) -.HP -\fB\-rpcconnect=\fR<ip> -.IP -Send commands to node running on <ip> (default: 127.0.0.1) -.HP -\fB\-rpccookiefile=\fR<loc> -.IP -Location of the auth cookie. Relative paths will be prefixed by a -net\-specific datadir location. (default: data dir) -.HP -\fB\-rpcpassword=\fR<pw> -.IP -Password for JSON\-RPC connections -.HP -\fB\-rpcport=\fR<port> -.IP -Connect to JSON\-RPC on <port> (default: 8332, testnet: 18332, regtest: -18443) -.HP -\fB\-rpcuser=\fR<user> -.IP -Username for JSON\-RPC connections -.HP -\fB\-rpcwait\fR -.IP -Wait for RPC server to start -.HP -\fB\-rpcwallet=\fR<walletname> -.IP -Send RPC for non\-default wallet on RPC server (needs to exactly match -corresponding \fB\-wallet\fR option passed to bitcoind). This changes -the RPC endpoint used, e.g. -http://127.0.0.1:8332/wallet/<walletname> -.HP -\fB\-stdin\fR -.IP -Read extra arguments from standard input, one per line until EOF/Ctrl\-D -(recommended for sensitive information such as passphrases). When -combined with \fB\-stdinrpcpass\fR, the first line from standard input -is used for the RPC password. -.HP -\fB\-stdinrpcpass\fR -.IP -Read RPC password from standard input as a single line. When combined -with \fB\-stdin\fR, the first line from standard input is used for the -RPC password. -.HP -\fB\-version\fR -.IP -Print version and exit -.PP -Chain selection options: -.HP -\fB\-testnet\fR -.IP -Use the test chain -.SH COPYRIGHT -Copyright (C) 2009-2019 The Bitcoin Core developers +bitcoin-cli \- manual page for bitcoin-cli -Please contribute if you find Bitcoin Core useful. Visit -<https://bitcoincore.org> for further information about the software. -The source code is available from <https://github.com/bitcoin/bitcoin>. - -This is experimental software. -Distributed under the MIT software license, see the accompanying file COPYING -or <https://opensource.org/licenses/MIT> +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index f68be21e8d..ff4d1d2c7a 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -1,610 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-QT "1" "February 2019" "bitcoin-qt v0.17.99.0" "User Commands" +.TH BITCOIN-QT "1" .SH NAME -bitcoin-qt \- manual page for bitcoin-qt v0.17.99.0 -.SH SYNOPSIS -.B bitcoin-qt -[\fI\,command-line options\/\fR] -.SH DESCRIPTION -Bitcoin Core version v0.17.99.0 (64\-bit) -.SH OPTIONS -.HP -\-? -.IP -Print this help message and exit -.HP -\fB\-alertnotify=\fR<cmd> -.IP -Execute command when a relevant alert is received or we see a really -long fork (%s in cmd is replaced by message) -.HP -\fB\-assumevalid=\fR<hex> -.IP -If this block is in the chain assume that it and its ancestors are valid -and potentially skip their script verification (0 to verify all, -default: -0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8, -testnet: -0000000000000037a8cd3e06cd5edbfe9dd1dbcc5dacab279376ef7cfc2b4c75) -.HP -\fB\-blocknotify=\fR<cmd> -.IP -Execute command when the best block changes (%s in cmd is replaced by -block hash) -.HP -\fB\-blockreconstructionextratxn=\fR<n> -.IP -Extra transactions to keep in memory for compact block reconstructions -(default: 100) -.HP -\fB\-blocksdir=\fR<dir> -.IP -Specify blocks directory (default: <datadir>/blocks) -.HP -\fB\-conf=\fR<file> -.IP -Specify configuration file. Relative paths will be prefixed by datadir -location. (default: bitcoin.conf) -.HP -\fB\-daemon\fR -.IP -Run in the background as a daemon and accept commands -.HP -\fB\-datadir=\fR<dir> -.IP -Specify data directory -.HP -\fB\-dbcache=\fR<n> -.IP -Set database cache size in MiB (4 to 16384, default: 450) -.HP -\fB\-debuglogfile=\fR<file> -.IP -Specify location of debug log file. Relative paths will be prefixed by a -net\-specific datadir location. (\fB\-nodebuglogfile\fR to disable; -default: debug.log) -.HP -\fB\-includeconf=\fR<file> -.IP -Specify additional configuration file, relative to the \fB\-datadir\fR path -(only useable from configuration file, not command line) -.HP -\fB\-loadblock=\fR<file> -.IP -Imports blocks from external blk000??.dat file on startup -.HP -\fB\-maxmempool=\fR<n> -.IP -Keep the transaction memory pool below <n> megabytes (default: 300) -.HP -\fB\-maxorphantx=\fR<n> -.IP -Keep at most <n> unconnectable transactions in memory (default: 100) -.HP -\fB\-mempoolexpiry=\fR<n> -.IP -Do not keep transactions in the mempool longer than <n> hours (default: -336) -.HP -\fB\-par=\fR<n> -.IP -Set the number of script verification threads (\fB\-8\fR to 16, 0 = auto, <0 = -leave that many cores free, default: 0) -.HP -\fB\-persistmempool\fR -.IP -Whether to save the mempool on shutdown and load on restart (default: 1) -.HP -\fB\-pid=\fR<file> -.IP -Specify pid file. Relative paths will be prefixed by a net\-specific -datadir location. (default: bitcoind.pid) -.HP -\fB\-prune=\fR<n> -.IP -Reduce storage requirements by enabling pruning (deleting) of old -blocks. This allows the pruneblockchain RPC to be called to -delete specific blocks, and enables automatic pruning of old -blocks if a target size in MiB is provided. This mode is -incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting this -setting requires re\-downloading the entire blockchain. (default: -0 = disable pruning blocks, 1 = allow manual pruning via RPC, ->=550 = automatically prune block files to stay under the -specified target size in MiB) -.HP -\fB\-reindex\fR -.IP -Rebuild chain state and block index from the blk*.dat files on disk -.HP -\fB\-reindex\-chainstate\fR -.IP -Rebuild chain state from the currently indexed blocks. When in pruning -mode or if blocks on disk might be corrupted, use full \fB\-reindex\fR -instead. -.HP -\fB\-sysperms\fR -.IP -Create new files with system default permissions, instead of umask 077 -(only effective with disabled wallet functionality) -.HP -\fB\-txindex\fR -.IP -Maintain a full transaction index, used by the getrawtransaction rpc -call (default: 0) -.HP -\fB\-version\fR -.IP -Print version and exit -.PP -Connection options: -.HP -\fB\-addnode=\fR<ip> -.IP -Add a node to connect to and attempt to keep the connection open (see -the `addnode` RPC command help for more info). This option can be -specified multiple times to add multiple nodes. -.HP -\fB\-banscore=\fR<n> -.IP -Threshold for disconnecting misbehaving peers (default: 100) -.HP -\fB\-bantime=\fR<n> -.IP -Number of seconds to keep misbehaving peers from reconnecting (default: -86400) -.HP -\fB\-bind=\fR<addr> -.IP -Bind to given address and always listen on it. Use [host]:port notation -for IPv6 -.HP -\fB\-connect=\fR<ip> -.IP -Connect only to the specified node; \fB\-noconnect\fR disables automatic -connections (the rules for this peer are the same as for -\fB\-addnode\fR). This option can be specified multiple times to connect -to multiple nodes. -.HP -\fB\-discover\fR -.IP -Discover own IP addresses (default: 1 when listening and no \fB\-externalip\fR -or \fB\-proxy\fR) -.HP -\fB\-dns\fR -.IP -Allow DNS lookups for \fB\-addnode\fR, \fB\-seednode\fR and \fB\-connect\fR (default: 1) -.HP -\fB\-dnsseed\fR -.IP -Query for peer addresses via DNS lookup, if low on addresses (default: 1 -unless \fB\-connect\fR used) -.HP -\fB\-externalip=\fR<ip> -.IP -Specify your own public address -.HP -\fB\-forcednsseed\fR -.IP -Always query for peer addresses via DNS lookup (default: 0) -.HP -\fB\-listen\fR -.IP -Accept connections from outside (default: 1 if no \fB\-proxy\fR or \fB\-connect\fR) -.HP -\fB\-listenonion\fR -.IP -Automatically create Tor hidden service (default: 1) -.HP -\fB\-maxconnections=\fR<n> -.IP -Maintain at most <n> connections to peers (default: 125) -.HP -\fB\-maxreceivebuffer=\fR<n> -.IP -Maximum per\-connection receive buffer, <n>*1000 bytes (default: 5000) -.HP -\fB\-maxsendbuffer=\fR<n> -.IP -Maximum per\-connection send buffer, <n>*1000 bytes (default: 1000) -.HP -\fB\-maxtimeadjustment\fR -.IP -Maximum allowed median peer time offset adjustment. Local perspective of -time may be influenced by peers forward or backward by this -amount. (default: 4200 seconds) -.HP -\fB\-maxuploadtarget=\fR<n> -.IP -Tries to keep outbound traffic under the given target (in MiB per 24h), -0 = no limit (default: 0) -.HP -\fB\-onion=\fR<ip:port> -.IP -Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -\fB\-noonion\fR to disable (default: \fB\-proxy\fR) -.HP -\fB\-onlynet=\fR<net> -.IP -Make outgoing connections only through network <net> (ipv4, ipv6 or -onion). Incoming connections are not affected by this option. -This option can be specified multiple times to allow multiple -networks. -.HP -\fB\-peerbloomfilters\fR -.IP -Support filtering of blocks and transaction with bloom filters (default: -1) -.HP -\fB\-permitbaremultisig\fR -.IP -Relay non\-P2SH multisig (default: 1) -.HP -\fB\-port=\fR<port> -.IP -Listen for connections on <port> (default: 8333, testnet: 18333, -regtest: 18444) -.HP -\fB\-proxy=\fR<ip:port> -.IP -Connect through SOCKS5 proxy, set \fB\-noproxy\fR to disable (default: -disabled) -.HP -\fB\-proxyrandomize\fR -.IP -Randomize credentials for every proxy connection. This enables Tor -stream isolation (default: 1) -.HP -\fB\-seednode=\fR<ip> -.IP -Connect to a node to retrieve peer addresses, and disconnect. This -option can be specified multiple times to connect to multiple -nodes. -.HP -\fB\-timeout=\fR<n> -.IP -Specify connection timeout in milliseconds (minimum: 1, default: 5000) -.HP -\fB\-torcontrol=\fR<ip>:<port> -.IP -Tor control port to use if onion listening enabled (default: -127.0.0.1:9051) -.HP -\fB\-torpassword=\fR<pass> -.IP -Tor control port password (default: empty) -.HP -\fB\-upnp\fR -.IP -Use UPnP to map the listening port (default: 0) -.HP -\fB\-whitebind=\fR<addr> -.IP -Bind to given address and whitelist peers connecting to it. Use -[host]:port notation for IPv6 -.HP -\fB\-whitelist=\fR<IP address or network> -.IP -Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or -CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple -times. Whitelisted peers cannot be DoS banned and their -transactions are always relayed, even if they are already in the -mempool, useful e.g. for a gateway -.PP -Wallet options: -.HP -\fB\-addresstype\fR -.IP -What type of addresses to use ("legacy", "p2sh\-segwit", or "bech32", -default: "p2sh\-segwit") -.HP -\fB\-avoidpartialspends\fR -.IP -Group outputs by address, selecting all or none, instead of selecting on -a per\-output basis. Privacy is improved as an address is only -used once (unless someone sends to it after spending from it), -but may result in slightly higher fees as suboptimal coin -selection may result due to the added limitation (default: 0) -.HP -\fB\-changetype\fR -.IP -What type of change to use ("legacy", "p2sh\-segwit", or "bech32"). -Default is same as \fB\-addresstype\fR, except when -\fB\-addresstype\fR=\fI\,p2sh\-segwit\/\fR a native segwit output is used when -sending to a native segwit address) -.HP -\fB\-disablewallet\fR -.IP -Do not load the wallet and disable wallet RPC calls -.HP -\fB\-discardfee=\fR<amt> -.IP -The fee rate (in BTC/kB) that indicates your tolerance for discarding -change by adding it to the fee (default: 0.0001). Note: An output -is discarded if it is dust at this rate, but we will always -discard up to the dust relay fee and a discard fee above that is -limited by the fee estimate for the longest target -.HP -\fB\-fallbackfee=\fR<amt> -.IP -A fee rate (in BTC/kB) that will be used when fee estimation has -insufficient data (default: 0.0002) -.HP -\fB\-keypool=\fR<n> -.IP -Set key pool size to <n> (default: 1000) -.HP -\fB\-mintxfee=\fR<amt> -.IP -Fees (in BTC/kB) smaller than this are considered zero fee for -transaction creation (default: 0.00001) -.HP -\fB\-paytxfee=\fR<amt> -.IP -Fee (in BTC/kB) to add to transactions you send (default: 0.00) -.HP -\fB\-rescan\fR -.IP -Rescan the block chain for missing wallet transactions on startup -.HP -\fB\-salvagewallet\fR -.IP -Attempt to recover private keys from a corrupt wallet on startup -.HP -\fB\-spendzeroconfchange\fR -.IP -Spend unconfirmed change when sending transactions (default: 1) -.HP -\fB\-txconfirmtarget=\fR<n> -.IP -If paytxfee is not set, include enough fee so transactions begin -confirmation on average within n blocks (default: 6) -.HP -\fB\-upgradewallet\fR -.IP -Upgrade wallet to latest format on startup -.HP -\fB\-wallet=\fR<path> -.IP -Specify wallet database path. Can be specified multiple times to load -multiple wallets. Path is interpreted relative to <walletdir> if -it is not absolute, and will be created if it does not exist (as -a directory containing a wallet.dat file and log files). For -backwards compatibility this will also accept names of existing -data files in <walletdir>.) -.HP -\fB\-walletbroadcast\fR -.IP -Make the wallet broadcast transactions (default: 1) -.HP -\fB\-walletdir=\fR<dir> -.IP -Specify directory to hold wallets (default: <datadir>/wallets if it -exists, otherwise <datadir>) -.HP -\fB\-walletnotify=\fR<cmd> -.IP -Execute command when a wallet transaction changes (%s in cmd is replaced -by TxID) -.HP -\fB\-walletrbf\fR -.IP -Send transactions with full\-RBF opt\-in enabled (RPC only, default: 0) -.HP -\fB\-zapwallettxes=\fR<mode> -.IP -Delete all wallet transactions and only recover those parts of the -blockchain through \fB\-rescan\fR on startup (1 = keep tx meta data e.g. -payment request information, 2 = drop tx meta data) -.PP -ZeroMQ notification options: -.HP -\fB\-zmqpubhashblock=\fR<address> -.IP -Enable publish hash block in <address> -.HP -\fB\-zmqpubhashtx=\fR<address> -.IP -Enable publish hash transaction in <address> -.HP -\fB\-zmqpubrawblock=\fR<address> -.IP -Enable publish raw block in <address> -.HP -\fB\-zmqpubrawtx=\fR<address> -.IP -Enable publish raw transaction in <address> -.PP -Debugging/Testing options: -.HP -\fB\-debug=\fR<category> -.IP -Output debugging information (default: \fB\-nodebug\fR, supplying <category> is -optional). If <category> is not supplied or if <category> = 1, -output all debugging information. <category> can be: net, tor, -mempool, http, bench, zmq, db, rpc, estimatefee, addrman, -selectcoins, reindex, cmpctblock, rand, prune, proxy, mempoolrej, -libevent, coindb, qt, leveldb. -.HP -\fB\-debugexclude=\fR<category> -.IP -Exclude debugging information for a category. Can be used in conjunction -with \fB\-debug\fR=\fI\,1\/\fR to output debug logs for all categories except one -or more specified categories. -.HP -\fB\-help\-debug\fR -.IP -Print help message with debugging options and exit -.HP -\fB\-logips\fR -.IP -Include IP addresses in debug output (default: 0) -.HP -\fB\-logtimestamps\fR -.IP -Prepend debug output with timestamp (default: 1) -.HP -\fB\-maxtxfee=\fR<amt> -.IP -Maximum total fees (in BTC) to use in a single wallet transaction or raw -transaction; setting this too low may abort large transactions -(default: 0.10) -.HP -\fB\-printtoconsole\fR -.IP -Send trace/debug info to console (default: 1 when no \fB\-daemon\fR. To disable -logging to file, set \fB\-nodebuglogfile\fR) -.HP -\fB\-shrinkdebugfile\fR -.IP -Shrink debug.log file on client startup (default: 1 when no \fB\-debug\fR) -.HP -\fB\-uacomment=\fR<cmt> -.IP -Append comment to the user agent string -.PP -Chain selection options: -.HP -\fB\-testnet\fR -.IP -Use the test chain -.PP -Node relay options: -.HP -\fB\-bytespersigop\fR -.IP -Equivalent bytes per sigop in transactions for relay and mining -(default: 20) -.HP -\fB\-datacarrier\fR -.IP -Relay and mine data carrier transactions (default: 1) -.HP -\fB\-datacarriersize\fR -.IP -Maximum size of data in data carrier transactions we relay and mine -(default: 83) -.HP -.HP -\fB\-minrelaytxfee=\fR<amt> -.IP -Fees (in BTC/kB) smaller than this are considered zero fee for relaying, -mining and transaction creation (default: 0.00001) -.HP -\fB\-whitelistforcerelay\fR -.IP -Force relay of transactions from whitelisted peers even if they violate -local relay policy (default: 0) -.HP -\fB\-whitelistrelay\fR -.IP -Accept relayed transactions received from whitelisted peers even when -not relaying transactions (default: 1) -.PP -Block creation options: -.HP -\fB\-blockmaxweight=\fR<n> -.IP -Set maximum BIP141 block weight (default: 3996000) -.HP -\fB\-blockmintxfee=\fR<amt> -.IP -Set lowest fee rate (in BTC/kB) for transactions to be included in block -creation. (default: 0.00001) -.PP -RPC server options: -.HP -\fB\-rest\fR -.IP -Accept public REST requests (default: 0) -.HP -\fB\-rpcallowip=\fR<ip> -.IP -Allow JSON\-RPC connections from specified source. Valid for <ip> are a -single IP (e.g. 1.2.3.4), a network/netmask (e.g. -1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This -option can be specified multiple times -.HP -\fB\-rpcauth=\fR<userpw> -.IP -Username and HMAC\-SHA\-256 hashed password for JSON\-RPC connections. The -field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A -canonical python script is included in share/rpcauth. The client -then connects normally using the -rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This -option can be specified multiple times -.HP -\fB\-rpcbind=\fR<addr>[:port] -.IP -Bind to given address to listen for JSON\-RPC connections. Do not expose -the RPC server to untrusted networks such as the public internet! -This option is ignored unless \fB\-rpcallowip\fR is also passed. Port is -optional and overrides \fB\-rpcport\fR. Use [host]:port notation for -IPv6. This option can be specified multiple times (default: -127.0.0.1 and ::1 i.e., localhost) -.HP -\fB\-rpccookiefile=\fR<loc> -.IP -Location of the auth cookie. Relative paths will be prefixed by a -net\-specific datadir location. (default: data dir) -.HP -\fB\-rpcpassword=\fR<pw> -.IP -Password for JSON\-RPC connections -.HP -\fB\-rpcport=\fR<port> -.IP -Listen for JSON\-RPC connections on <port> (default: 8332, testnet: -18332, regtest: 18443) -.HP -\fB\-rpcserialversion\fR -.IP -Sets the serialization of raw transaction or block hex returned in -non\-verbose mode, non\-segwit(0) or segwit(1) (default: 1) -.HP -\fB\-rpcthreads=\fR<n> -.IP -Set the number of threads to service RPC calls (default: 4) -.HP -\fB\-rpcuser=\fR<user> -.IP -Username for JSON\-RPC connections -.HP -\fB\-server\fR -.IP -Accept command line and JSON\-RPC commands -.PP -UI Options: -.HP -\fB\-choosedatadir\fR -.IP -Choose data directory on startup (default: 0) -.HP -\fB\-lang=\fR<lang> -.IP -Set language, for example "de_DE" (default: system locale) -.HP -\fB\-min\fR -.IP -Start minimized -.HP -\fB\-resetguisettings\fR -.IP -Reset all settings changed in the GUI -.HP -\fB\-rootcertificates=\fR<file> -.IP -Set SSL root certificates for payment request (default: \fB\-system\-\fR) -.HP -\fB\-splash\fR -.IP -Show splash screen on startup (default: 1) -.SH COPYRIGHT -Copyright (C) 2009-2019 The Bitcoin Core developers +bitcoin-qt \- manual page for bitcoin-qt -Please contribute if you find Bitcoin Core useful. Visit -<https://bitcoincore.org> for further information about the software. -The source code is available from <https://github.com/bitcoin/bitcoin>. - -This is experimental software. -Distributed under the MIT software license, see the accompanying file COPYING -or <https://opensource.org/licenses/MIT> +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index b4c7698896..776bb46234 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -1,116 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-TX "1" "February 2019" "bitcoin-tx v0.17.99.0" "User Commands" +.TH BITCOIN-TX "1" .SH NAME -bitcoin-tx \- manual page for bitcoin-tx v0.17.99.0 -.SH SYNOPSIS -.B bitcoin-tx -[\fI\,options\/\fR] \fI\,<hex-tx> \/\fR[\fI\,commands\/\fR] \fI\,Update hex-encoded bitcoin transaction\/\fR -.br -.B bitcoin-tx -[\fI\,options\/\fR] \fI\,-create \/\fR[\fI\,commands\/\fR] \fI\,Create hex-encoded bitcoin transaction\/\fR -.SH DESCRIPTION -Bitcoin Core bitcoin\-tx utility version v0.17.99.0 -.SH OPTIONS -.HP -\-? -.IP -Print this help message and exit -.HP -\fB\-create\fR -.IP -Create new, empty TX. -.HP -\fB\-json\fR -.IP -Select JSON output -.HP -\fB\-txid\fR -.IP -Output only the hex\-encoded transaction id of the resultant transaction. -.PP -Chain selection options: -.HP -\fB\-testnet\fR -.IP -Use the test chain -.PP -Commands: -.IP -delin=N -.IP -Delete input N from TX -.IP -delout=N -.IP -Delete output N from TX -.IP -in=TXID:VOUT(:SEQUENCE_NUMBER) -.IP -Add input to TX -.IP -locktime=N -.IP -Set TX lock time to N -.IP -nversion=N -.IP -Set TX version to N -.IP -outaddr=VALUE:ADDRESS -.IP -Add address\-based output to TX -.IP -outdata=[VALUE:]DATA -.IP -Add data\-based output to TX -.IP -outmultisig=VALUE:REQUIRED:PUBKEYS:PUBKEY1:PUBKEY2:....[:FLAGS] -.IP -Add Pay To n\-of\-m Multi\-sig output to TX. n = REQUIRED, m = PUBKEYS. -Optionally add the "W" flag to produce a -pay\-to\-witness\-script\-hash output. Optionally add the "S" flag to -wrap the output in a pay\-to\-script\-hash. -.IP -outpubkey=VALUE:PUBKEY[:FLAGS] -.IP -Add pay\-to\-pubkey output to TX. Optionally add the "W" flag to produce a -pay\-to\-witness\-pubkey\-hash output. Optionally add the "S" flag to -wrap the output in a pay\-to\-script\-hash. -.IP -outscript=VALUE:SCRIPT[:FLAGS] -.IP -Add raw script output to TX. Optionally add the "W" flag to produce a -pay\-to\-witness\-script\-hash output. Optionally add the "S" flag to -wrap the output in a pay\-to\-script\-hash. -.IP -replaceable(=N) -.IP -Set RBF opt\-in sequence number for input N (if not provided, opt\-in all -available inputs) -.IP -sign=SIGHASH\-FLAGS -.IP -Add zero or more signatures to transaction. This command requires JSON -registers:prevtxs=JSON object, privatekeys=JSON object. See -signrawtransactionwithkey docs for format of sighash flags, JSON -objects. -.PP -Register Commands: -.IP -load=NAME:FILENAME -.IP -Load JSON file FILENAME into register NAME -.IP -set=NAME:JSON\-STRING -.IP -Set register NAME to given JSON\-STRING -.SH COPYRIGHT -Copyright (C) 2009-2019 The Bitcoin Core developers +bitcoin-tx \- manual page for bitcoin-tx -Please contribute if you find Bitcoin Core useful. Visit -<https://bitcoincore.org> for further information about the software. -The source code is available from <https://github.com/bitcoin/bitcoin>. - -This is experimental software. -Distributed under the MIT software license, see the accompanying file COPYING -or <https://opensource.org/licenses/MIT> +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-util.1 b/doc/man/bitcoin-util.1 new file mode 100644 index 0000000000..5c733c6e21 --- /dev/null +++ b/doc/man/bitcoin-util.1 @@ -0,0 +1,5 @@ +.TH BITCOIN-UTIL "1" +.SH NAME +bitcoin-util \- manual page for bitcoin-util + +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 index aadea09a2b..2da43dec66 100644 --- a/doc/man/bitcoin-wallet.1 +++ b/doc/man/bitcoin-wallet.1 @@ -1,63 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-WALLET "1" "February 2019" "bitcoin-wallet v0.17.99.0" "User Commands" +.TH BITCOIN-WALLET "1" .SH NAME -bitcoin-wallet \- manual page for bitcoin-wallet v0.17.99.0 -.SH DESCRIPTION -Bitcoin Core bitcoin\-wallet version v0.17.99.0 -.PP -wallet\-tool is an offline tool for creating and interacting with Bitcoin Core wallet files. -By default wallet\-tool will act on wallets in the default mainnet wallet directory in the datadir. -To change the target wallet, use the \fB\-datadir\fR, \fB\-wallet\fR and \fB\-testnet\fR/\-regtest arguments. -.SS "Usage:" -.IP -bitcoin\-wallet [options] <command> -.SH OPTIONS -.HP -\-? -.IP -Print this help message and exit -.HP -\fB\-datadir=\fR<dir> -.IP -Specify data directory -.HP -\fB\-wallet=\fR<wallet\-name> -.IP -Specify wallet name -.PP -Debugging/Testing options: -.HP -\fB\-debug=\fR<category> -.IP -Output debugging information (default: 0). -.HP -\fB\-printtoconsole\fR -.IP -Send trace/debug info to console (default: 1 when no \fB\-debug\fR is true, 0 -otherwise. -.PP -Chain selection options: -.HP -\fB\-testnet\fR -.IP -Use the test chain -.PP -Commands: -.IP -create -.IP -Create new wallet file -.IP -info -.IP -Get wallet info -.SH COPYRIGHT -Copyright (C) 2009-2019 The Bitcoin Core developers +bitcoin-wallet \- manual page for bitcoin-wallet -Please contribute if you find Bitcoin Core useful. Visit -<https://bitcoincore.org> for further information about the software. -The source code is available from <https://github.com/bitcoin/bitcoin>. - -This is experimental software. -Distributed under the MIT software license, see the accompanying file COPYING -or <https://opensource.org/licenses/MIT> +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index 211ba10285..2c88f74520 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -1,583 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIND "1" "February 2019" "bitcoind v0.17.99.0" "User Commands" +.TH BITCOIND "1" .SH NAME -bitcoind \- manual page for bitcoind v0.17.99.0 -.SH SYNOPSIS -.B bitcoind -[\fI\,options\/\fR] \fI\,Start Bitcoin Core Daemon\/\fR -.SH DESCRIPTION -Bitcoin Core Daemon version v0.17.99.0 -.SH OPTIONS -.HP -\-? -.IP -Print this help message and exit -.HP -\fB\-alertnotify=\fR<cmd> -.IP -Execute command when a relevant alert is received or we see a really -long fork (%s in cmd is replaced by message) -.HP -\fB\-assumevalid=\fR<hex> -.IP -If this block is in the chain assume that it and its ancestors are valid -and potentially skip their script verification (0 to verify all, -default: -0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8, -testnet: -0000000000000037a8cd3e06cd5edbfe9dd1dbcc5dacab279376ef7cfc2b4c75) -.HP -\fB\-blocknotify=\fR<cmd> -.IP -Execute command when the best block changes (%s in cmd is replaced by -block hash) -.HP -\fB\-blockreconstructionextratxn=\fR<n> -.IP -Extra transactions to keep in memory for compact block reconstructions -(default: 100) -.HP -\fB\-blocksdir=\fR<dir> -.IP -Specify blocks directory (default: <datadir>/blocks) -.HP -\fB\-conf=\fR<file> -.IP -Specify configuration file. Relative paths will be prefixed by datadir -location. (default: bitcoin.conf) -.HP -\fB\-daemon\fR -.IP -Run in the background as a daemon and accept commands -.HP -\fB\-datadir=\fR<dir> -.IP -Specify data directory -.HP -\fB\-dbcache=\fR<n> -.IP -Set database cache size in MiB (4 to 16384, default: 450) -.HP -\fB\-debuglogfile=\fR<file> -.IP -Specify location of debug log file. Relative paths will be prefixed by a -net\-specific datadir location. (\fB\-nodebuglogfile\fR to disable; -default: debug.log) -.HP -\fB\-includeconf=\fR<file> -.IP -Specify additional configuration file, relative to the \fB\-datadir\fR path -(only useable from configuration file, not command line) -.HP -\fB\-loadblock=\fR<file> -.IP -Imports blocks from external blk000??.dat file on startup -.HP -\fB\-maxmempool=\fR<n> -.IP -Keep the transaction memory pool below <n> megabytes (default: 300) -.HP -\fB\-maxorphantx=\fR<n> -.IP -Keep at most <n> unconnectable transactions in memory (default: 100) -.HP -\fB\-mempoolexpiry=\fR<n> -.IP -Do not keep transactions in the mempool longer than <n> hours (default: -336) -.HP -\fB\-par=\fR<n> -.IP -Set the number of script verification threads (\fB\-8\fR to 16, 0 = auto, <0 = -leave that many cores free, default: 0) -.HP -\fB\-persistmempool\fR -.IP -Whether to save the mempool on shutdown and load on restart (default: 1) -.HP -\fB\-pid=\fR<file> -.IP -Specify pid file. Relative paths will be prefixed by a net\-specific -datadir location. (default: bitcoind.pid) -.HP -\fB\-prune=\fR<n> -.IP -Reduce storage requirements by enabling pruning (deleting) of old -blocks. This allows the pruneblockchain RPC to be called to -delete specific blocks, and enables automatic pruning of old -blocks if a target size in MiB is provided. This mode is -incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting this -setting requires re\-downloading the entire blockchain. (default: -0 = disable pruning blocks, 1 = allow manual pruning via RPC, ->=550 = automatically prune block files to stay under the -specified target size in MiB) -.HP -\fB\-reindex\fR -.IP -Rebuild chain state and block index from the blk*.dat files on disk -.HP -\fB\-reindex\-chainstate\fR -.IP -Rebuild chain state from the currently indexed blocks. When in pruning -mode or if blocks on disk might be corrupted, use full \fB\-reindex\fR -instead. -.HP -\fB\-sysperms\fR -.IP -Create new files with system default permissions, instead of umask 077 -(only effective with disabled wallet functionality) -.HP -\fB\-txindex\fR -.IP -Maintain a full transaction index, used by the getrawtransaction rpc -call (default: 0) -.HP -\fB\-version\fR -.IP -Print version and exit -.PP -Connection options: -.HP -\fB\-addnode=\fR<ip> -.IP -Add a node to connect to and attempt to keep the connection open (see -the `addnode` RPC command help for more info). This option can be -specified multiple times to add multiple nodes. -.HP -\fB\-banscore=\fR<n> -.IP -Threshold for disconnecting misbehaving peers (default: 100) -.HP -\fB\-bantime=\fR<n> -.IP -Number of seconds to keep misbehaving peers from reconnecting (default: -86400) -.HP -\fB\-bind=\fR<addr> -.IP -Bind to given address and always listen on it. Use [host]:port notation -for IPv6 -.HP -\fB\-connect=\fR<ip> -.IP -Connect only to the specified node; \fB\-noconnect\fR disables automatic -connections (the rules for this peer are the same as for -\fB\-addnode\fR). This option can be specified multiple times to connect -to multiple nodes. -.HP -\fB\-discover\fR -.IP -Discover own IP addresses (default: 1 when listening and no \fB\-externalip\fR -or \fB\-proxy\fR) -.HP -\fB\-dns\fR -.IP -Allow DNS lookups for \fB\-addnode\fR, \fB\-seednode\fR and \fB\-connect\fR (default: 1) -.HP -\fB\-dnsseed\fR -.IP -Query for peer addresses via DNS lookup, if low on addresses (default: 1 -unless \fB\-connect\fR used) -.HP -\fB\-externalip=\fR<ip> -.IP -Specify your own public address -.HP -\fB\-forcednsseed\fR -.IP -Always query for peer addresses via DNS lookup (default: 0) -.HP -\fB\-listen\fR -.IP -Accept connections from outside (default: 1 if no \fB\-proxy\fR or \fB\-connect\fR) -.HP -\fB\-listenonion\fR -.IP -Automatically create Tor hidden service (default: 1) -.HP -\fB\-maxconnections=\fR<n> -.IP -Maintain at most <n> connections to peers (default: 125) -.HP -\fB\-maxreceivebuffer=\fR<n> -.IP -Maximum per\-connection receive buffer, <n>*1000 bytes (default: 5000) -.HP -\fB\-maxsendbuffer=\fR<n> -.IP -Maximum per\-connection send buffer, <n>*1000 bytes (default: 1000) -.HP -\fB\-maxtimeadjustment\fR -.IP -Maximum allowed median peer time offset adjustment. Local perspective of -time may be influenced by peers forward or backward by this -amount. (default: 4200 seconds) -.HP -\fB\-maxuploadtarget=\fR<n> -.IP -Tries to keep outbound traffic under the given target (in MiB per 24h), -0 = no limit (default: 0) -.HP -\fB\-onion=\fR<ip:port> -.IP -Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -\fB\-noonion\fR to disable (default: \fB\-proxy\fR) -.HP -\fB\-onlynet=\fR<net> -.IP -Make outgoing connections only through network <net> (ipv4, ipv6 or -onion). Incoming connections are not affected by this option. -This option can be specified multiple times to allow multiple -networks. -.HP -\fB\-peerbloomfilters\fR -.IP -Support filtering of blocks and transaction with bloom filters (default: -1) -.HP -\fB\-permitbaremultisig\fR -.IP -Relay non\-P2SH multisig (default: 1) -.HP -\fB\-port=\fR<port> -.IP -Listen for connections on <port> (default: 8333, testnet: 18333, -regtest: 18444) -.HP -\fB\-proxy=\fR<ip:port> -.IP -Connect through SOCKS5 proxy, set \fB\-noproxy\fR to disable (default: -disabled) -.HP -\fB\-proxyrandomize\fR -.IP -Randomize credentials for every proxy connection. This enables Tor -stream isolation (default: 1) -.HP -\fB\-seednode=\fR<ip> -.IP -Connect to a node to retrieve peer addresses, and disconnect. This -option can be specified multiple times to connect to multiple -nodes. -.HP -\fB\-timeout=\fR<n> -.IP -Specify connection timeout in milliseconds (minimum: 1, default: 5000) -.HP -\fB\-torcontrol=\fR<ip>:<port> -.IP -Tor control port to use if onion listening enabled (default: -127.0.0.1:9051) -.HP -\fB\-torpassword=\fR<pass> -.IP -Tor control port password (default: empty) -.HP -\fB\-upnp\fR -.IP -Use UPnP to map the listening port (default: 0) -.HP -\fB\-whitebind=\fR<addr> -.IP -Bind to given address and whitelist peers connecting to it. Use -[host]:port notation for IPv6 -.HP -\fB\-whitelist=\fR<IP address or network> -.IP -Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or -CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple -times. Whitelisted peers cannot be DoS banned and their -transactions are always relayed, even if they are already in the -mempool, useful e.g. for a gateway -.PP -Wallet options: -.HP -\fB\-addresstype\fR -.IP -What type of addresses to use ("legacy", "p2sh\-segwit", or "bech32", -default: "p2sh\-segwit") -.HP -\fB\-avoidpartialspends\fR -.IP -Group outputs by address, selecting all or none, instead of selecting on -a per\-output basis. Privacy is improved as an address is only -used once (unless someone sends to it after spending from it), -but may result in slightly higher fees as suboptimal coin -selection may result due to the added limitation (default: 0) -.HP -\fB\-changetype\fR -.IP -What type of change to use ("legacy", "p2sh\-segwit", or "bech32"). -Default is same as \fB\-addresstype\fR, except when -\fB\-addresstype\fR=\fI\,p2sh\-segwit\/\fR a native segwit output is used when -sending to a native segwit address) -.HP -\fB\-disablewallet\fR -.IP -Do not load the wallet and disable wallet RPC calls -.HP -\fB\-discardfee=\fR<amt> -.IP -The fee rate (in BTC/kB) that indicates your tolerance for discarding -change by adding it to the fee (default: 0.0001). Note: An output -is discarded if it is dust at this rate, but we will always -discard up to the dust relay fee and a discard fee above that is -limited by the fee estimate for the longest target -.HP -\fB\-fallbackfee=\fR<amt> -.IP -A fee rate (in BTC/kB) that will be used when fee estimation has -insufficient data (default: 0.0002) -.HP -\fB\-keypool=\fR<n> -.IP -Set key pool size to <n> (default: 1000) -.HP -\fB\-mintxfee=\fR<amt> -.IP -Fees (in BTC/kB) smaller than this are considered zero fee for -transaction creation (default: 0.00001) -.HP -\fB\-paytxfee=\fR<amt> -.IP -Fee (in BTC/kB) to add to transactions you send (default: 0.00) -.HP -\fB\-rescan\fR -.IP -Rescan the block chain for missing wallet transactions on startup -.HP -\fB\-salvagewallet\fR -.IP -Attempt to recover private keys from a corrupt wallet on startup -.HP -\fB\-spendzeroconfchange\fR -.IP -Spend unconfirmed change when sending transactions (default: 1) -.HP -\fB\-txconfirmtarget=\fR<n> -.IP -If paytxfee is not set, include enough fee so transactions begin -confirmation on average within n blocks (default: 6) -.HP -\fB\-upgradewallet\fR -.IP -Upgrade wallet to latest format on startup -.HP -\fB\-wallet=\fR<path> -.IP -Specify wallet database path. Can be specified multiple times to load -multiple wallets. Path is interpreted relative to <walletdir> if -it is not absolute, and will be created if it does not exist (as -a directory containing a wallet.dat file and log files). For -backwards compatibility this will also accept names of existing -data files in <walletdir>.) -.HP -\fB\-walletbroadcast\fR -.IP -Make the wallet broadcast transactions (default: 1) -.HP -\fB\-walletdir=\fR<dir> -.IP -Specify directory to hold wallets (default: <datadir>/wallets if it -exists, otherwise <datadir>) -.HP -\fB\-walletnotify=\fR<cmd> -.IP -Execute command when a wallet transaction changes (%s in cmd is replaced -by TxID) -.HP -\fB\-walletrbf\fR -.IP -Send transactions with full\-RBF opt\-in enabled (RPC only, default: 0) -.HP -\fB\-zapwallettxes=\fR<mode> -.IP -Delete all wallet transactions and only recover those parts of the -blockchain through \fB\-rescan\fR on startup (1 = keep tx meta data e.g. -payment request information, 2 = drop tx meta data) -.PP -ZeroMQ notification options: -.HP -\fB\-zmqpubhashblock=\fR<address> -.IP -Enable publish hash block in <address> -.HP -\fB\-zmqpubhashtx=\fR<address> -.IP -Enable publish hash transaction in <address> -.HP -\fB\-zmqpubrawblock=\fR<address> -.IP -Enable publish raw block in <address> -.HP -\fB\-zmqpubrawtx=\fR<address> -.IP -Enable publish raw transaction in <address> -.PP -Debugging/Testing options: -.HP -\fB\-debug=\fR<category> -.IP -Output debugging information (default: \fB\-nodebug\fR, supplying <category> is -optional). If <category> is not supplied or if <category> = 1, -output all debugging information. <category> can be: net, tor, -mempool, http, bench, zmq, db, rpc, estimatefee, addrman, -selectcoins, reindex, cmpctblock, rand, prune, proxy, mempoolrej, -libevent, coindb, qt, leveldb. -.HP -\fB\-debugexclude=\fR<category> -.IP -Exclude debugging information for a category. Can be used in conjunction -with \fB\-debug\fR=\fI\,1\/\fR to output debug logs for all categories except one -or more specified categories. -.HP -\fB\-help\-debug\fR -.IP -Print help message with debugging options and exit -.HP -\fB\-logips\fR -.IP -Include IP addresses in debug output (default: 0) -.HP -\fB\-logtimestamps\fR -.IP -Prepend debug output with timestamp (default: 1) -.HP -\fB\-maxtxfee=\fR<amt> -.IP -Maximum total fees (in BTC) to use in a single wallet transaction or raw -transaction; setting this too low may abort large transactions -(default: 0.10) -.HP -\fB\-printtoconsole\fR -.IP -Send trace/debug info to console (default: 1 when no \fB\-daemon\fR. To disable -logging to file, set \fB\-nodebuglogfile\fR) -.HP -\fB\-shrinkdebugfile\fR -.IP -Shrink debug.log file on client startup (default: 1 when no \fB\-debug\fR) -.HP -\fB\-uacomment=\fR<cmt> -.IP -Append comment to the user agent string -.PP -Chain selection options: -.HP -\fB\-testnet\fR -.IP -Use the test chain -.PP -Node relay options: -.HP -\fB\-bytespersigop\fR -.IP -Equivalent bytes per sigop in transactions for relay and mining -(default: 20) -.HP -\fB\-datacarrier\fR -.IP -Relay and mine data carrier transactions (default: 1) -.HP -\fB\-datacarriersize\fR -.IP -Maximum size of data in data carrier transactions we relay and mine -(default: 83) -.HP -\fB\-minrelaytxfee=\fR<amt> -.IP -Fees (in BTC/kB) smaller than this are considered zero fee for relaying, -mining and transaction creation (default: 0.00001) -.HP -\fB\-whitelistforcerelay\fR -.IP -Force relay of transactions from whitelisted peers even if they violate -local relay policy (default: 0) -.HP -\fB\-whitelistrelay\fR -.IP -Accept relayed transactions received from whitelisted peers even when -not relaying transactions (default: 1) -.PP -Block creation options: -.HP -\fB\-blockmaxweight=\fR<n> -.IP -Set maximum BIP141 block weight (default: 3996000) -.HP -\fB\-blockmintxfee=\fR<amt> -.IP -Set lowest fee rate (in BTC/kB) for transactions to be included in block -creation. (default: 0.00001) -.PP -RPC server options: -.HP -\fB\-rest\fR -.IP -Accept public REST requests (default: 0) -.HP -\fB\-rpcallowip=\fR<ip> -.IP -Allow JSON\-RPC connections from specified source. Valid for <ip> are a -single IP (e.g. 1.2.3.4), a network/netmask (e.g. -1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This -option can be specified multiple times -.HP -\fB\-rpcauth=\fR<userpw> -.IP -Username and HMAC\-SHA\-256 hashed password for JSON\-RPC connections. The -field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A -canonical python script is included in share/rpcauth. The client -then connects normally using the -rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This -option can be specified multiple times -.HP -\fB\-rpcbind=\fR<addr>[:port] -.IP -Bind to given address to listen for JSON\-RPC connections. Do not expose -the RPC server to untrusted networks such as the public internet! -This option is ignored unless \fB\-rpcallowip\fR is also passed. Port is -optional and overrides \fB\-rpcport\fR. Use [host]:port notation for -IPv6. This option can be specified multiple times (default: -127.0.0.1 and ::1 i.e., localhost) -.HP -\fB\-rpccookiefile=\fR<loc> -.IP -Location of the auth cookie. Relative paths will be prefixed by a -net\-specific datadir location. (default: data dir) -.HP -\fB\-rpcpassword=\fR<pw> -.IP -Password for JSON\-RPC connections -.HP -\fB\-rpcport=\fR<port> -.IP -Listen for JSON\-RPC connections on <port> (default: 8332, testnet: -18332, regtest: 18443) -.HP -\fB\-rpcserialversion\fR -.IP -Sets the serialization of raw transaction or block hex returned in -non\-verbose mode, non\-segwit(0) or segwit(1) (default: 1) -.HP -\fB\-rpcthreads=\fR<n> -.IP -Set the number of threads to service RPC calls (default: 4) -.HP -\fB\-rpcuser=\fR<user> -.IP -Username for JSON\-RPC connections -.HP -\fB\-server\fR -.IP -Accept command line and JSON\-RPC commands -.SH COPYRIGHT -Copyright (C) 2009-2019 The Bitcoin Core developers +bitcoind \- manual page for bitcoind -Please contribute if you find Bitcoin Core useful. Visit -<https://bitcoincore.org> for further information about the software. -The source code is available from <https://github.com/bitcoin/bitcoin>. - -This is experimental software. -Distributed under the MIT software license, see the accompanying file COPYING -or <https://opensource.org/licenses/MIT> +This is a placeholder file. Please follow the instructions in \fIcontrib/devtools/README.md\fR to generate the manual pages after a release. diff --git a/doc/productivity.md b/doc/productivity.md index 555f0afe3c..a01c6f545d 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -51,6 +51,7 @@ After running `./autogen.sh`, which generates the `./configure` file, use `./con ```sh --without-miniupnpc +--without-natpmp --disable-bench --disable-wallet --without-gui diff --git a/doc/release-notes-18077.md b/doc/release-notes-18077.md new file mode 100644 index 0000000000..4034a4c19c --- /dev/null +++ b/doc/release-notes-18077.md @@ -0,0 +1,10 @@ +P2P and network changes +----------------------- + +- Added NAT-PMP port mapping support via [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html) + +Command-line options +-------------------- + +- The `-natpmp` option has been added to use NAT-PMP to map the listening port. If both UPnP +and NAT-PMP are enabled, a successful allocation from UPnP prevails over one from NAT-PMP. diff --git a/doc/release-notes-19776.md b/doc/release-notes-19776.md new file mode 100644 index 0000000000..5553c5a7bd --- /dev/null +++ b/doc/release-notes-19776.md @@ -0,0 +1,9 @@ +Updated RPCs +------------ + +- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and + `bip152_hb_from`, that respectively indicate whether we selected a peer to be + in compact blocks high-bandwidth mode or whether a peer selected us as a + compact blocks high-bandwidth peer. High-bandwidth peers send new block + announcements via a `cmpctblock` message rather than the usual inv/headers + announcements. See BIP 152 for more details. (#19776) diff --git a/doc/release-notes.md b/doc/release-notes.md index 0331328ff5..8f1e03e16b 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -45,437 +45,66 @@ wallet versions of Bitcoin Core are generally supported. Compatibility ============== -During this release cycle, work has been done to ensure that the codebase is fully -compatible with C++17. The intention is to begin using C++17 features starting -with the 0.22.0 release. This means that a compiler that supports C++17 will be -required to compile 0.22.0. - Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. -From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +From Bitcoin Core 0.22.0 onwards, macOS versions earlier than 10.14 are no longer supported. Additionally, Bitcoin Core does not yet change appearance when macOS "dark mode" is activated. -The node's known peers are persisted to disk in a file called `peers.dat`. The -format of this file has been changed in a backwards-incompatible way in order to -accommodate the storage of Tor v3 and other BIP155 addresses. This means that if -the file is modified by 0.21.0 or newer then older versions will not be able to -read it. Those old versions, in the event of a downgrade, will log an error -message "Incorrect keysize in addrman deserialization" and will continue normal -operation as if the file was missing, creating a new empty one. (#19954) - Notable changes =============== P2P and network changes ----------------------- -- The mempool now tracks whether transactions submitted via the wallet or RPCs - have been successfully broadcast. Every 10-15 minutes, the node will try to - announce unbroadcast transactions until a peer requests it via a `getdata` - message or the transaction is removed from the mempool for other reasons. - The node will not track the broadcast status of transactions submitted to the - node using P2P relay. This version reduces the initial broadcast guarantees - for wallet transactions submitted via P2P to a node running the wallet. (#18038) - -- The size of the set of transactions that peers have announced and we consider - for requests has been reduced from 100000 to 5000 (per peer), and further - announcements will be ignored when that limit is reached. If you need to dump - (very) large batches of transactions, exceptions can be made for trusted - peers using the "relay" network permission. For localhost for example it can - be enabled using the command line option `-whitelist=relay@127.0.0.1`. - (#19988) - -- The Tor onion service that is automatically created by setting the - `-listenonion` configuration parameter will now be created as a Tor v3 service - instead of Tor v2. The private key that was used for Tor v2 (if any) will be - left untouched in the `onion_private_key` file in the data directory (see - `-datadir`) and can be removed if not needed. Bitcoin Core will no longer - attempt to read it. The private key for the Tor v3 service will be saved in a - file named `onion_v3_private_key`. To use the deprecated Tor v2 service (not - recommended), then `onion_private_key` can be copied over - `onion_v3_private_key`, e.g. - `cp -f onion_private_key onion_v3_private_key`. (#19954) - Updated RPCs ------------ - -- The `getpeerinfo` RPC now has additional `last_block` and `last_transaction` - fields that return the UNIX epoch time of the last block and the last valid - transaction received from each peer. (#19731) - -- `getnetworkinfo` now returns two new fields, `connections_in` and - `connections_out`, that provide the number of inbound and outbound peer - connections. These new fields are in addition to the existing `connections` - field, which returns the total number of peer connections. (#19405) - -- Exposed transaction version numbers are now treated as unsigned 32-bit - integers instead of signed 32-bit integers. This matches their treatment in - consensus logic. Versions greater than 2 continue to be non-standard - (matching previous behavior of smaller than 1 or greater than 2 being - non-standard). Note that this includes the joinpsbt command, which combines - partially-signed transactions by selecting the highest version number. - (#16525) - -- `getmempoolinfo` now returns an additional `unbroadcastcount` field. The - mempool tracks locally submitted transactions until their initial broadcast - is acknowledged by a peer. This field returns the count of transactions - waiting for acknowledgement. - -- Mempool RPCs such as `getmempoolentry` and `getrawmempool` with - `verbose=true` now return an additional `unbroadcast` field. This indicates - whether initial broadcast of the transaction has been acknowledged by a - peer. `getmempoolancestors` and `getmempooldescendants` are also updated. - -- The `bumpfee`, `fundrawtransaction`, `sendmany`, `sendtoaddress`, and `walletcreatefundedpsbt` -RPC commands have been updated to include two new fee estimation methods "BTC/kB" and "sat/B". -The target is the fee expressed explicitly in the given form. Note that use of this feature -will trigger BIP 125 (replace-by-fee) opt-in. (#11413) - -- In addition, the `estimate_mode` parameter is now case insensitive for all of - the above RPC commands. (#11413) - -- The `bumpfee` command now uses `conf_target` rather than `confTarget` in the - options. (#11413) - -- The `getpeerinfo` RPC no longer returns the `banscore` field unless the configuration - option `-deprecatedrpc=banscore` is used. The `banscore` field will be fully - removed in the next major release. (#19469) - -- The `testmempoolaccept` RPC returns `vsize` and a `fees` object with the `base` fee - if the transaction would pass validation. (#19940) - -- The `getpeerinfo` RPC now returns a `connection_type` field. This indicates - the type of connection established with the peer. It will return one of six - options. For more information, see the `getpeerinfo` help documentation. - (#19725) - -- The `getpeerinfo` RPC no longer returns the `addnode` field by default. This - field will be fully removed in the next major release. It can be accessed - with the configuration option `-deprecatedrpc=getpeerinfo_addnode`. However, - it is recommended to instead use the `connection_type` field (it will return - `manual` when addnode is true). (#19725) - -- The `walletcreatefundedpsbt` RPC call will now fail with - `Insufficient funds` when inputs are manually selected but are not enough to cover - the outputs and fee. Additional inputs can automatically be added through the - new `add_inputs` option. (#16377) - -- The `fundrawtransaction` RPC now supports `add_inputs` option that when `false` - prevents adding more inputs if necessary and consequently the RPC fails. +- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, + and `whitelisted`, which were previously deprecated in 0.21. Instead of + `addnode`, the `connection_type` field returns manual. Instead of + `whitelisted`, the `permissions` field indicates if the peer has special + privileges. The `banscore` field has simply been removed. (#20755) Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. New RPCs -------- -- The `getindexinfo` RPC returns the actively running indices of the node, - including their current sync status and height. It also accepts an `index_name` - to specify returning only the status of that index. (#19550) - Build System ------------ +New settings +------------ + Updated settings ---------------- -- The same ZeroMQ notification (e.g. `-zmqpubhashtx=address`) can now be - specified multiple times to publish the same notification to different ZeroMQ - sockets. (#18309) - -- The `-banscore` configuration option, which modified the default threshold for - disconnecting and discouraging misbehaving peers, has been removed as part of - changes in 0.20.1 and in this release to the handling of misbehaving peers. - Refer to "Changes regarding misbehaving peers" in the 0.20.1 release notes for - details. (#19464) - -- The `-debug=db` logging category, which was deprecated in 0.20 and replaced by - `-debug=walletdb` to distinguish it from `coindb`, has been removed. (#19202) +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. -- A `download` permission has been extracted from the `noban` permission. For - compatibility, `noban` implies the `download` permission, but this may change - in future releases. Refer to the help of the affected settings `-whitebind` - and `-whitelist` for more details. (#19191) - -- Netmasks that contain 1-bits after 0-bits (the 1-bits are not contiguous on - the left side, e.g. 255.0.255.255) are no longer accepted. They are invalid - according to RFC 4632. Netmasks are used in the `-rpcallowip` and `-whitelist` - configuration options and in the `setban` RPC. (#19628) - -Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. +- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) Tools and Utilities ------------------- -- The `connections` field of `bitcoin-cli -getinfo` is expanded to return a JSON - object with `in`, `out` and `total` numbers of peer connections. It previously - returned a single integer value for the total number of peer connections. (#19405) - -- A new `bitcoin-cli -generate` command, equivalent to RPC `generatenewaddress` - followed by `generatetoaddress`, can generate blocks for command line testing - purposes. This is a client-side version of the - former `generate` RPC. See the help for details. (#19133) - -- The `bitcoin-cli -getinfo` command now displays the wallet name and balance for - each of the loaded wallets when more than one is loaded (e.g. in multiwallet - mode) and a wallet is not specified with `-rpcwallet`. (#18594) - -New settings ------------- - -- The `startupnotify` option is used to specify a command to - execute when Bitcoin Core has finished with its startup - sequence. (#15367) - Wallet ------ -- Backwards compatibility has been dropped for two `getaddressinfo` RPC - deprecations, as notified in the 0.20 release notes. The deprecated `label` - field has been removed as well as the deprecated `labels` behavior of - returning a JSON object containing `name` and `purpose` key-value pairs. Since - 0.20, the `labels` field returns a JSON array of label names. (#19200) - -- To improve wallet privacy, the frequency of wallet rebroadcast attempts is - reduced from approximately once every 15 minutes to once every 12-36 hours. - To maintain a similar level of guarantee for initial broadcast of wallet - transactions, the mempool tracks these transactions as a part of the newly - introduced unbroadcast set. See the "P2P and network changes" section for - more information on the unbroadcast set. (#18038) - -- The `sendtoaddress` and `sendmany` RPCs accept an optional `verbose=True` - argument to also return the fee reason about the sent tx. (#19501) - -- The wallet can create a transaction without change even when the keypool is - empty. Previously it failed. (#17219) - -- The `-salvagewallet` startup option has been removed. A new `salvage` command - has been added to the `bitcoin-wallet` tool which performs the salvage - operations that `-salvagewallet` did. (#18918) - -- A new configuration flag `-maxapsfee` has been added, which sets the max - allowed avoid partial spends (APS) fee. It defaults to 0 (i.e. fee is the - same with and without APS). Setting it to -1 will disable APS, unless - `-avoidpartialspends` is set. (#14582) - -- The wallet will now avoid partial spends (APS) by default, if this does not - result in a difference in fees compared to the non-APS variant. The allowed - fee threshold can be adjusted using the new `-maxapsfee` configuration - option. (#14582) - -- The `createwallet`, `loadwallet`, and `unloadwallet` RPCs now accept - `load_on_startup` options to modify the settings list. Unless these options - are explicitly set to true or false, the list is not modified, so the RPC - methods remain backwards compatible. (#15937) - -- A new `send` RPC with similar syntax to `walletcreatefundedpsbt`, including - support for coin selection and a custom fee rate. The `send` RPC is - experimental and may change in subsequent releases. Using it is encouraged - once it's no longer experimental: `sendmany` and `sendtoaddress` may be - deprecated in a future release. (#16378) - -- `fundrawtransaction` and `walletcreatefundedpsbt` when used with the - `lockUnspents` argument now lock manually selected coins, in addition to - automatically selected coins. Note that locked coins are never used in - automatic coin selection, but can still be manually selected. (#18244) - -- The `-zapwallettxes` startup option has been removed and its functionality - removed from the wallet. This option was originally intended to allow for - the fee bumping of transactions that did not signal RBF. This functionality - has been superseded with the abandon transaction feature. (#19671) - -- The error code when no wallet is loaded, but a wallet RPC is called, has been - changed from `-32601` (method not found) to `-18` (wallet not found). - (#20101) - -### Automatic wallet creation removed - -Bitcoin Core will no longer automatically create new wallets on startup. It will -load existing wallets specified by `-wallet` options on the command line or in -`bitcoin.conf` or `settings.json` files. And by default it will also load a -top-level unnamed ("") wallet. However, if specified wallets don't exist, -Bitcoin Core will now just log warnings instead of creating new wallets with -new keys and addresses like previous releases did. - -New wallets can be created through the GUI (which has a more prominent create -wallet option), through the `bitcoin-cli createwallet` or `bitcoin-wallet -create` commands, or the `createwallet` RPC. (#15454) - -### Experimental Descriptor Wallets - -Please note that Descriptor Wallets are still experimental and not all expected functionality -is available. Additionally there may be some bugs and current functions may change in the future. -Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). - -0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store -scriptPubKey information using descriptors. This is in contrast to the Legacy Wallet -structure where keys are used to generate scriptPubKeys and addresses. Because of this -shift to being script based instead of key based, many of the confusing things that Legacy -Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition -of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. -Descriptor Wallets also uses different semantics for watch-only things and imports. - -As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. -Users who already have a Bitcoin Core wallet can continue to use it as they did before without -any change in behavior. Newly created Legacy Wallets (which is the default type of wallet) will -behave as they did in previous versions of Bitcoin Core. - -The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing -things. They are intended to behave similarly except for the import/export and watchonly functionality -as described below. - -#### Creating Descriptor Wallets - -Descriptor Wallets are not created by default. They must be explicitly created using the -`createwallet` RPC or via the GUI. A `descriptors` option has been added to `createwallet`. -Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. - -In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a -Descriptor Wallet should be created. - -Without those options being set, a Legacy Wallet will be created instead. Additionally the -Default Wallet created upon first startup of Bitcoin Core will be a Legacy Wallet. - -#### `IsMine` Semantics - -`IsMine` refers to the function used to determine whether a script belongs to the wallet. -This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets -returns true if the wallet would be able to sign an input that spends an output with that script. -Since keys can be involved in a variety of different scripts, this definition for `IsMine` can -lead to many unexpected scripts being considered part of the wallet. - -With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by -the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly -what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` -in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` -allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type -(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type -even without that address being requested from the wallet. Descriptor Wallets does not -allow for this and will only watch for the addresses that were explicitly requested from the wallet. - -These changes to `IsMine` will make it easier to reason about what scripts the wallet will -actually be watching for in outputs. However for the vast majority of users, this change is -largely transparent and will not have noticeable effect. - -#### Imports and Exports - -In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts -and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` -logic as it has to distinguish between spendable and watchonly. - -Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be -imported. These descriptors are then added to the wallet as if it were a descriptor generated by -the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish -between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also -different and described in more detail in the next section. - -To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax -similar to that of `importmulti`. - -As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys -the existing import RPCs have been disabled for descriptor wallets. -New export RPCs for Descriptor Wallets have not yet been added. - -The following RPCs are disabled for Descriptor Wallets: - -* importprivkey -* importpubkey -* importaddress -* importwallet -* dumpprivkey -* dumpwallet -* importmulti -* addmultisigaddress -* sethdseed - -#### Watchonly Wallets - -A Legacy Wallet contains both private keys and scripts that were being watched. -Those watched scripts would not contribute to your normal balance. In order to see the watchonly -balance and to use watchonly things in transactions, an `include_watchonly` option was added -to many RPCs that would allow users to do that. However it is easy to forget to include this option. - -Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be -watchonly depending on whether it was created with private keys disabled. This eliminates the need -to distinguish between things that are watchonly and things that are not within a wallet itself. - -This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has -a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), -then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT -workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just -non-functional. - -This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such -multiple key descriptors as some transactions could be signed and broadast and others not. This is -due to some transactions containing only single key inputs, while others would contain both single -key and multiple key inputs, depending on which are available and how the coin selection algorithm -selects inputs. However this is not considered to be a supported use case; multisigs -should be in their own wallets which do not already have descriptors. Although users cannot export -descriptors with private keys for now as explained earlier. - -#### BIP 44/49/84 Support - -The change to using descriptors changes the default derivation paths used by Bitcoin Core -to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without -issue. - -### Wallet RPC changes - -- The `upgradewallet` RPC replaces the `-upgradewallet` command line option. - (#15761) -- The `settxfee` RPC will fail if the fee was set higher than the `-maxtxfee` - command line setting. The wallet will already fail to create transactions - with fees higher than `-maxtxfee`. (#18467) - GUI changes ----------- -- Wallets created or loaded in the GUI will now be automatically loaded on - startup, so they don't need to be manually reloaded next time Bitcoin Core is - started. The list of wallets to load on startup is stored in - `\<datadir\>/settings.json` and augments any command line or `bitcoin.conf` - `-wallet=` settings that specify more wallets to load. Wallets that are - unloaded in the GUI get removed from the settings list so they won't load - again automatically next startup. (#19754) - -- The GUI Peers window no longer displays a "Ban Score" field. This is part of - changes in 0.20.1 and in this release to the handling of misbehaving - peers. Refer to "Changes regarding misbehaving peers" in the 0.20.1 release - notes for details. (#19512) - Low-level changes ================= RPC --- -- To make RPC `sendtoaddress` more consistent with `sendmany` the following error - `sendtoaddress` codes were changed from `-4` to `-6`: - - Insufficient funds - - Fee estimation failed - - Transaction has too long of a mempool chain - -- The `sendrawtransaction` error code for exceeding `maxfeerate` has been changed from - `-26` to `-25`. The error string has been changed from "absurdly-high-fee" to - "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)." The - `testmempoolaccept` RPC returns `max-fee-exceeded` rather than `absurdly-high-fee` - as the `reject-reason`. (#19339) - -- To make wallet and rawtransaction RPCs more consistent, the error message for - exceeding maximum feerate has been changed to "Fee exceeds maximum configured by user - (e.g. -maxtxfee, maxfeerate)." (#19339) - Tests ----- -- The BIP 325 default signet can be enabled by the `-chain=signet` or `-signet` - setting. The settings `-signetchallenge` and `-signetseednode` allow - enabling a custom signet. - Credits ======= diff --git a/doc/release-process.md b/doc/release-process.md index 8191c3034d..92845bcc82 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -18,10 +18,11 @@ Release Process ### Before every major release * On both the master branch and the new release branch: - - update `CLIENT_VERSION_MINOR` in [`configure.ac`](../configure.ac) - - update `CLIENT_VERSION_MINOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) + - update `CLIENT_VERSION_MAJOR` in [`configure.ac`](../configure.ac) + - update `CLIENT_VERSION_MAJOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) * On the new release branch in [`configure.ac`](../configure.ac) and [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): - - set `CLIENT_VERSION_REVISION` to `0` + - set `CLIENT_VERSION_MINOR` to `0` + - set `CLIENT_VERSION_BUILD` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` #### Before branch-off @@ -306,9 +307,9 @@ bitcoin.org (see below for bitcoin.org update instructions). - First, check to see if the Bitcoin.org maintainers have prepared a release: https://github.com/bitcoin-dot-org/bitcoin.org/labels/Core - - If they have, it will have previously failed their Travis CI + - If they have, it will have previously failed their CI checks because the final release files weren't uploaded. - Trigger a Travis CI rebuild---if it passes, merge. + Trigger a CI rebuild---if it passes, merge. - If they have not prepared a release, follow the Bitcoin.org release instructions: https://github.com/bitcoin-dot-org/bitcoin.org/blob/master/docs/adding-events-release-notes-and-alerts.md#release-notes diff --git a/doc/shared-libraries.md b/doc/shared-libraries.md index e960863a80..147e223711 100644 --- a/doc/shared-libraries.md +++ b/doc/shared-libraries.md @@ -41,9 +41,10 @@ The interface is defined in the C header `bitcoinconsensus.h` located in `src/sc - `bitcoinconsensus_ERR_TX_SIZE_MISMATCH` - `txToLen` did not match with the size of `txTo` - `bitcoinconsensus_ERR_DESERIALIZE` - An error deserializing `txTo` - `bitcoinconsensus_ERR_AMOUNT_REQUIRED` - Input amount is required if WITNESS is used +- `bitcoinconsensus_ERR_INVALID_FLAGS` - Script verification `flags` are invalid (i.e. not part of the libconsensus interface) ### Example Implementations -- [NBitcoin](https://github.com/NicolasDorier/NBitcoin/blob/master/NBitcoin/Script.cs#L814) (.NET Bindings) +- [NBitcoin](https://github.com/MetacoSA/NBitcoin/blob/5e1055cd7c4186dee4227c344af8892aea54faec/NBitcoin/Script.cs#L979-#L1031) (.NET Bindings) - [node-libbitcoinconsensus](https://github.com/bitpay/node-libbitcoinconsensus) (Node.js Bindings) - [java-libbitcoinconsensus](https://github.com/dexX7/java-libbitcoinconsensus) (Java Bindings) - [bitcoinconsensus-php](https://github.com/Bit-Wasp/bitcoinconsensus-php) (PHP Bindings) diff --git a/doc/tor.md b/doc/tor.md index 12b5f70245..1ba7137b8e 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -5,6 +5,16 @@ It is possible to run Bitcoin Core as a Tor onion service, and connect to such s The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly configure Tor. +## How to see information about your Tor configuration via Bitcoin Core + +There are several ways to see your local onion address in Bitcoin Core: +- in the debug log (grep for "tor:" or "AddLocal") +- in the output of RPC `getnetworkinfo` in the "localaddresses" section +- in the output of the CLI `-netinfo` peer connections dashboard + +You may set the `-debug=tor` config logging option to have additional +information in the debug log about your Tor configuration. + ## 1. Run Bitcoin Core behind a Tor proxy @@ -37,7 +47,7 @@ In a typical situation, this suffices to run behind a Tor proxy: ./bitcoind -proxy=127.0.0.1:9050 -## 2. Run a Bitcoin Core hidden server +## 2. Manually create a Bitcoin Core onion service If you configure your Tor system accordingly, it is possible to make your node also reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent @@ -46,20 +56,24 @@ versions of Tor see [Section 3](#3-automatically-listen-on-tor).* HiddenServiceDir /var/lib/tor/bitcoin-service/ HiddenServicePort 8333 127.0.0.1:8334 - HiddenServicePort 18333 127.0.0.1:18334 The directory can be different of course, but virtual port numbers should be equal to your bitcoind's P2P listen port (8333 by default), and target addresses and ports should be equal to binding address and port for inbound Tor connections (127.0.0.1:8334 by default). - -externalip=X You can tell bitcoin about its publicly reachable address using - this option, and this can be a .onion address. Given the above - configuration, you can find your .onion address in + -externalip=X You can tell bitcoin about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in /var/lib/tor/bitcoin-service/hostname. For connections coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), .onion addresses are given + Tor proxy typically runs), onion addresses are given preference for your node to advertise itself with. + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + -listen You'll need to enable listening for incoming connections, as this is off by default behind a proxy. @@ -72,7 +86,7 @@ should be equal to binding address and port for inbound Tor connections (127.0.0 In a typical situation, where you're only reachable via Tor, this should suffice: - ./bitcoind -proxy=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -listen + ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing @@ -85,14 +99,14 @@ as well, use `discover` instead: ./bitcoind ... -discover -and open port 8333 on your firewall (or use -upnp). +and open port 8333 on your firewall (or use port mapping, i.e., `-upnp` or `-natpmp`). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./bitcoind -onion=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -discover + ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover -## 3. Automatically listen on Tor +## 3. Automatically create a Bitcoin Core onion service Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket API, to create and destroy 'ephemeral' onion services programmatically. @@ -130,6 +144,6 @@ in the tor configuration file. The hashed password can be obtained with the comm - Do not add anything but Bitcoin Core ports to the onion service created in section 2. If you run a web service too, create a new onion service for that. - Otherwise it is trivial to link them, which may reduce privacy. Hidden + Otherwise it is trivial to link them, which may reduce privacy. Onion services created automatically (as in section 3) always have only one port open. diff --git a/share/genbuild.sh b/share/genbuild.sh index 1153df933f..fe89ae1fde 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ fi # otherwise generate suffix from git, i.e. string like "59887e8-dirty" - GIT_COMMIT=$(git rev-parse --short HEAD) + GIT_COMMIT=$(git rev-parse --short=12 HEAD) git diff-index --quiet HEAD -- || GIT_COMMIT="$GIT_COMMIT-dirty" fi diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 3befba3425..207d60aca3 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.12.0</string> + <string>10.14.0</string> <key>LSArchitecturePriority</key> <array> @@ -17,13 +17,13 @@ <string>APPL</string> <key>NSHumanReadableCopyright</key> - <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string> + <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string> <key>CFBundleShortVersionString</key> - <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@</string> + <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@</string> <key>CFBundleVersion</key> - <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@</string> + <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@</string> <key>CFBundleSignature</key> <string>????</string> diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 5431909bb2..681f243d04 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -56,7 +56,7 @@ CRCCheck on XPStyle on BrandingText " " ShowInstDetails show -VIProductVersion @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@ +VIProductVersion @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@.0 VIAddVersionKey ProductName "@PACKAGE_NAME@" VIAddVersionKey ProductVersion "@PACKAGE_VERSION@" VIAddVersionKey CompanyName "${COMPANY}" diff --git a/src/Makefile.am b/src/Makefile.am index 67fd402603..2871df124c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,9 +20,8 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a @@ -93,15 +92,21 @@ endif if BUILD_BITCOIN_CLI bin_PROGRAMS += bitcoin-cli endif + if BUILD_BITCOIN_TX bin_PROGRAMS += bitcoin-tx endif + if ENABLE_WALLET if BUILD_BITCOIN_WALLET bin_PROGRAMS += bitcoin-wallet endif endif +if BUILD_BITCOIN_UTIL + bin_PROGRAMS += bitcoin-util +endif + .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ @@ -153,6 +158,7 @@ BITCOIN_CORE_H = \ key_io.h \ logging.h \ logging/timer.h \ + mapport.h \ memusage.h \ merkleblock.h \ miner.h \ @@ -224,6 +230,7 @@ BITCOIN_CORE_H = \ util/error.h \ util/fees.h \ util/golombrice.h \ + util/hasher.h \ util/macros.h \ util/memory.h \ util/message.h \ @@ -250,6 +257,7 @@ BITCOIN_CORE_H = \ wallet/context.h \ wallet/crypter.h \ wallet/db.h \ + wallet/dump.h \ wallet/feebumper.h \ wallet/fees.h \ wallet/ismine.h \ @@ -299,14 +307,14 @@ libbitcoin_server_a_SOURCES = \ index/blockfilterindex.cpp \ index/txindex.cpp \ init.cpp \ - interfaces/chain.cpp \ - interfaces/node.cpp \ + mapport.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ node/coin.cpp \ node/coinstats.cpp \ node/context.cpp \ + node/interfaces.cpp \ node/psbt.cpp \ node/transaction.cpp \ node/ui_interface.cpp \ @@ -359,18 +367,17 @@ endif libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ - interfaces/wallet.cpp \ - wallet/bdb.cpp \ wallet/coincontrol.cpp \ wallet/context.cpp \ wallet/crypter.cpp \ wallet/db.cpp \ + wallet/dump.cpp \ wallet/feebumper.cpp \ wallet/fees.cpp \ + wallet/interfaces.cpp \ wallet/load.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ - wallet/salvage.cpp \ wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ @@ -381,6 +388,9 @@ libbitcoin_wallet_a_SOURCES = \ if USE_SQLITE libbitcoin_wallet_a_SOURCES += wallet/sqlite.cpp endif +if USE_BDB +libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp +endif libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -407,6 +417,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/hmac_sha512.h \ crypto/poly1305.h \ crypto/poly1305.cpp \ + crypto/muhash.h \ + crypto/muhash.cpp \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ @@ -539,6 +551,7 @@ libbitcoin_util_a_SOURCES = \ util/bytevectorhash.cpp \ util/error.cpp \ util/fees.cpp \ + util/hasher.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ @@ -595,7 +608,7 @@ bitcoin_bin_ldadd = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) +bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) bitcoind_SOURCES = $(bitcoin_daemon_sources) bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) @@ -661,6 +674,27 @@ bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc endif # +# bitcoin-util binary # +bitcoin_util_SOURCES = bitcoin-util.cpp +bitcoin_util_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_util_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_util_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +if TARGET_WINDOWS +bitcoin_util_SOURCES += bitcoin-util-res.rc +endif + +bitcoin_util_LDADD = \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) + +bitcoin_util_LDADD += $(BOOST_LIBS) +# + # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h @@ -741,13 +775,13 @@ endif if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) + $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif if EMBEDDED_LEVELDB diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index beb3f8dfd2..56b8ca8ce6 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -74,7 +74,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 802b3a2e4b..113272e65e 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -41,7 +41,7 @@ crc32c_libcrc32c_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crc32c_libcrc32c_a_SOURCES = crc32c_libcrc32c_a_SOURCES += crc32c/include/crc32c/crc32c.h crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_linux_check.h +crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_check.h crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_internal.h crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_prefetch.h crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_read_le.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index f46310a603..3d41d203d3 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -320,7 +320,7 @@ if ENABLE_ZMQ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index c05dd38737..a6a857d952 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -55,7 +55,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 9cc383c240..e9f9b73abe 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,171 +2,15 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -FUZZ_TARGETS = \ - test/fuzz/addition_overflow \ - test/fuzz/addr_info_deserialize \ - test/fuzz/addrdb \ - test/fuzz/address_deserialize \ - test/fuzz/addrman \ - test/fuzz/addrman_deserialize \ - test/fuzz/asmap \ - test/fuzz/asmap_direct \ - test/fuzz/autofile \ - test/fuzz/banentry_deserialize \ - test/fuzz/banman \ - test/fuzz/base_encode_decode \ - test/fuzz/bech32 \ - test/fuzz/block \ - test/fuzz/block_deserialize \ - test/fuzz/block_file_info_deserialize \ - test/fuzz/block_filter_deserialize \ - test/fuzz/block_header \ - test/fuzz/block_header_and_short_txids_deserialize \ - test/fuzz/blockfilter \ - test/fuzz/blockheader_deserialize \ - test/fuzz/blocklocator_deserialize \ - test/fuzz/blockmerkleroot \ - test/fuzz/blocktransactions_deserialize \ - test/fuzz/blocktransactionsrequest_deserialize \ - test/fuzz/blockundo_deserialize \ - test/fuzz/bloom_filter \ - test/fuzz/bloomfilter_deserialize \ - test/fuzz/buffered_file \ - test/fuzz/chain \ - test/fuzz/checkqueue \ - test/fuzz/coins_deserialize \ - test/fuzz/coins_view \ - test/fuzz/connman \ - test/fuzz/crypto \ - test/fuzz/crypto_aes256 \ - test/fuzz/crypto_aes256cbc \ - test/fuzz/crypto_chacha20 \ - test/fuzz/crypto_chacha20_poly1305_aead \ - test/fuzz/crypto_common \ - test/fuzz/crypto_hkdf_hmac_sha256_l32 \ - test/fuzz/crypto_poly1305 \ - test/fuzz/cuckoocache \ - test/fuzz/decode_tx \ - test/fuzz/descriptor_parse \ - test/fuzz/diskblockindex_deserialize \ - test/fuzz/eval_script \ - test/fuzz/fee_rate \ - test/fuzz/fee_rate_deserialize \ - test/fuzz/fees \ - test/fuzz/flat_file_pos_deserialize \ - test/fuzz/flatfile \ - test/fuzz/float \ - test/fuzz/golomb_rice \ - test/fuzz/hex \ - test/fuzz/http_request \ - test/fuzz/integer \ - test/fuzz/inv_deserialize \ - test/fuzz/key \ - test/fuzz/key_io \ - test/fuzz/key_origin_info_deserialize \ - test/fuzz/kitchen_sink \ - test/fuzz/load_external_block_file \ - test/fuzz/locale \ - test/fuzz/merkle_block_deserialize \ - test/fuzz/merkleblock \ - test/fuzz/message \ - test/fuzz/messageheader_deserialize \ - test/fuzz/multiplication_overflow \ - test/fuzz/net \ - test/fuzz/net_permissions \ - test/fuzz/netaddr_deserialize \ - test/fuzz/netaddress \ - test/fuzz/out_point_deserialize \ - test/fuzz/p2p_transport_deserializer \ - test/fuzz/parse_hd_keypath \ - test/fuzz/parse_iso8601 \ - test/fuzz/parse_numbers \ - test/fuzz/parse_script \ - test/fuzz/parse_univalue \ - test/fuzz/partial_merkle_tree_deserialize \ - test/fuzz/partially_signed_transaction_deserialize \ - test/fuzz/policy_estimator \ - test/fuzz/policy_estimator_io \ - test/fuzz/pow \ - test/fuzz/prefilled_transaction_deserialize \ - test/fuzz/prevector \ - test/fuzz/primitives_transaction \ - test/fuzz/process_message \ - test/fuzz/process_message_addr \ - test/fuzz/process_message_block \ - test/fuzz/process_message_blocktxn \ - test/fuzz/process_message_cmpctblock \ - test/fuzz/process_message_feefilter \ - test/fuzz/process_message_filteradd \ - test/fuzz/process_message_filterclear \ - test/fuzz/process_message_filterload \ - test/fuzz/process_message_getaddr \ - test/fuzz/process_message_getblocks \ - test/fuzz/process_message_getblocktxn \ - test/fuzz/process_message_getdata \ - test/fuzz/process_message_getheaders \ - test/fuzz/process_message_headers \ - test/fuzz/process_message_inv \ - test/fuzz/process_message_mempool \ - test/fuzz/process_message_notfound \ - test/fuzz/process_message_ping \ - test/fuzz/process_message_pong \ - test/fuzz/process_message_sendcmpct \ - test/fuzz/process_message_sendheaders \ - test/fuzz/process_message_tx \ - test/fuzz/process_message_verack \ - test/fuzz/process_message_version \ - test/fuzz/process_messages \ - test/fuzz/protocol \ - test/fuzz/psbt \ - test/fuzz/psbt_input_deserialize \ - test/fuzz/psbt_output_deserialize \ - test/fuzz/pub_key_deserialize \ - test/fuzz/random \ - test/fuzz/rbf \ - test/fuzz/rolling_bloom_filter \ - test/fuzz/script \ - test/fuzz/script_bitcoin_consensus \ - test/fuzz/script_descriptor_cache \ - test/fuzz/script_deserialize \ - test/fuzz/script_flags \ - test/fuzz/script_interpreter \ - test/fuzz/script_assets_test_minimizer \ - test/fuzz/script_ops \ - test/fuzz/script_sigcache \ - test/fuzz/script_sign \ - test/fuzz/scriptnum_ops \ - test/fuzz/secp256k1_ec_seckey_import_export_der \ - test/fuzz/secp256k1_ecdsa_signature_parse_der_lax \ - test/fuzz/service_deserialize \ - test/fuzz/signature_checker \ - test/fuzz/signet \ - test/fuzz/snapshotmetadata_deserialize \ - test/fuzz/span \ - test/fuzz/spanparsing \ - test/fuzz/string \ - test/fuzz/strprintf \ - test/fuzz/sub_net_deserialize \ - test/fuzz/system \ - test/fuzz/timedata \ - test/fuzz/transaction \ - test/fuzz/tx_in \ - test/fuzz/tx_in_deserialize \ - test/fuzz/tx_out \ - test/fuzz/txoutcompressor_deserialize \ - test/fuzz/txrequest \ - test/fuzz/txundo_deserialize \ - test/fuzz/uint160_deserialize \ - test/fuzz/uint256_deserialize - if ENABLE_FUZZ -noinst_PROGRAMS += $(FUZZ_TARGETS:=) +noinst_PROGRAMS += test/fuzz/fuzz else bin_PROGRAMS += test/test_bitcoin endif TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) +FUZZ_BINARY=test/fuzz/fuzz$(EXEEXT) JSON_TEST_FILES = \ test/data/script_tests.json \ @@ -292,15 +136,19 @@ BITCOIN_TESTS =\ if ENABLE_WALLET BITCOIN_TESTS += \ - wallet/test/db_tests.cpp \ wallet/test/psbt_wallet_tests.cpp \ wallet/test/wallet_tests.cpp \ + wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ wallet/test/coinselector_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ wallet/test/scriptpubkeyman_tests.cpp +if USE_BDB +BITCOIN_TESTS += wallet/test/db_tests.cpp +endif + BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ @@ -319,7 +167,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) +test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ @@ -330,935 +178,106 @@ if ENABLE_FUZZ FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -test_fuzz_addition_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_addition_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addition_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addition_overflow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_addition_overflow_SOURCES = test/fuzz/addition_overflow.cpp - -test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 -test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addr_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_addr_info_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_addrdb_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_addrdb_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addrdb_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrdb_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_addrdb_SOURCES = test/fuzz/addrdb.cpp - -test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 -test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_address_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_address_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_addrman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_addrman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addrman_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrman_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_addrman_SOURCES = test/fuzz/addrman.cpp - -test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 -test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrman_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_addrman_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_asmap_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_asmap_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_asmap_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_asmap_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_asmap_SOURCES = test/fuzz/asmap.cpp - -test_fuzz_asmap_direct_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_asmap_direct_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_asmap_direct_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_asmap_direct_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_asmap_direct_SOURCES = test/fuzz/asmap_direct.cpp - -test_fuzz_autofile_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_autofile_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_autofile_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_autofile_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_autofile_SOURCES = test/fuzz/autofile.cpp - -test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 -test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_banentry_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_banentry_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_banman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_banman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_banman_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_banman_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_banman_SOURCES = test/fuzz/banman.cpp - -test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_base_encode_decode_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_base_encode_decode_SOURCES = test/fuzz/base_encode_decode.cpp - -test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bech32_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_bech32_SOURCES = test/fuzz/bech32.cpp - -test_fuzz_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_SOURCES = test/fuzz/block.cpp - -test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 -test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 -test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_file_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_file_info_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 -test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_filter_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_filter_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_block_header_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_block_header_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_header_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_header_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_header_SOURCES = test/fuzz/block_header.cpp - -test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 -test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_block_header_and_short_txids_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blockfilter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_blockfilter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockfilter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockfilter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blockfilter_SOURCES = test/fuzz/blockfilter.cpp - -test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 -test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockheader_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blockheader_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 -test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocklocator_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blocklocator_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 -test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockmerkleroot_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blockmerkleroot_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 -test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactions_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blocktransactions_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 -test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 -test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockundo_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_blockundo_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bloom_filter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_bloom_filter_SOURCES = test/fuzz/bloom_filter.cpp - -test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 -test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bloomfilter_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_bloomfilter_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_buffered_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_buffered_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_buffered_file_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_buffered_file_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_buffered_file_SOURCES = test/fuzz/buffered_file.cpp - -test_fuzz_chain_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_chain_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_chain_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_chain_SOURCES = test/fuzz/chain.cpp - -test_fuzz_checkqueue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_checkqueue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_checkqueue_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_checkqueue_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_checkqueue_SOURCES = test/fuzz/checkqueue.cpp - -test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 -test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_coins_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_coins_view_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_coins_view_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_coins_view_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp - -test_fuzz_connman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_connman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_connman_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_connman_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_connman_SOURCES = test/fuzz/connman.cpp - -test_fuzz_crypto_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_SOURCES = test/fuzz/crypto.cpp - -test_fuzz_crypto_aes256_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_aes256_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_aes256_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_aes256_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_aes256_SOURCES = test/fuzz/crypto_aes256.cpp - -test_fuzz_crypto_aes256cbc_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_aes256cbc_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_aes256cbc_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_aes256cbc_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_aes256cbc_SOURCES = test/fuzz/crypto_aes256cbc.cpp - -test_fuzz_crypto_chacha20_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_chacha20_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_chacha20_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_chacha20_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_chacha20_SOURCES = test/fuzz/crypto_chacha20.cpp - -test_fuzz_crypto_chacha20_poly1305_aead_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_chacha20_poly1305_aead_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_chacha20_poly1305_aead_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_chacha20_poly1305_aead_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_chacha20_poly1305_aead_SOURCES = test/fuzz/crypto_chacha20_poly1305_aead.cpp - -test_fuzz_crypto_common_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_common_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_common_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_common_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_common_SOURCES = test/fuzz/crypto_common.cpp - -test_fuzz_crypto_hkdf_hmac_sha256_l32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_hkdf_hmac_sha256_l32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_hkdf_hmac_sha256_l32_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_hkdf_hmac_sha256_l32_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_hkdf_hmac_sha256_l32_SOURCES = test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp - -test_fuzz_crypto_poly1305_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_crypto_poly1305_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_crypto_poly1305_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_poly1305_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_crypto_poly1305_SOURCES = test/fuzz/crypto_poly1305.cpp - -test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_cuckoocache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_cuckoocache_SOURCES = test/fuzz/cuckoocache.cpp - -test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_decode_tx_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_decode_tx_SOURCES = test/fuzz/decode_tx.cpp - -test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_descriptor_parse_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_descriptor_parse_SOURCES = test/fuzz/descriptor_parse.cpp - -test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 -test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_diskblockindex_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_diskblockindex_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_eval_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_eval_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_eval_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_eval_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_eval_script_SOURCES = test/fuzz/eval_script.cpp - -test_fuzz_fee_rate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_fee_rate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_fee_rate_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fee_rate_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_fee_rate_SOURCES = test/fuzz/fee_rate.cpp - -test_fuzz_fee_rate_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFEE_RATE_DESERIALIZE=1 -test_fuzz_fee_rate_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fee_rate_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_fee_rate_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_fees_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_fees_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_fees_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fees_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_fees_SOURCES = test/fuzz/fees.cpp - -test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 -test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_flat_file_pos_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_flatfile_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_flatfile_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_flatfile_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_flatfile_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_flatfile_SOURCES = test/fuzz/flatfile.cpp - -test_fuzz_float_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_float_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_float_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_float_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_float_SOURCES = test/fuzz/float.cpp - -test_fuzz_golomb_rice_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_golomb_rice_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_golomb_rice_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_golomb_rice_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_golomb_rice_SOURCES = test/fuzz/golomb_rice.cpp - -test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_hex_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_hex_SOURCES = test/fuzz/hex.cpp - -test_fuzz_http_request_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_http_request_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_http_request_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_http_request_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_http_request_SOURCES = test/fuzz/http_request.cpp - -test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_integer_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_integer_SOURCES = test/fuzz/integer.cpp - -test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 -test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_inv_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_inv_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_key_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_key_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_key_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_key_SOURCES = test/fuzz/key.cpp - -test_fuzz_key_io_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_key_io_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_key_io_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_io_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_key_io_SOURCES = test/fuzz/key_io.cpp - -test_fuzz_key_origin_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DKEY_ORIGIN_INFO_DESERIALIZE=1 -test_fuzz_key_origin_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_origin_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_key_origin_info_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_kitchen_sink_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_kitchen_sink_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_kitchen_sink_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp - -test_fuzz_load_external_block_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_load_external_block_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_load_external_block_file_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_load_external_block_file_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_load_external_block_file_SOURCES = test/fuzz/load_external_block_file.cpp - -test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_locale_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_locale_SOURCES = test/fuzz/locale.cpp - -test_fuzz_merkle_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMERKLE_BLOCK_DESERIALIZE=1 -test_fuzz_merkle_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_merkle_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_merkle_block_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_merkle_block_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_merkleblock_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_merkleblock_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_merkleblock_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_merkleblock_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_merkleblock_SOURCES = test/fuzz/merkleblock.cpp - -test_fuzz_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_message_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_message_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_message_SOURCES = test/fuzz/message.cpp - -test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 -test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_messageheader_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_messageheader_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_multiplication_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_multiplication_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_multiplication_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_multiplication_overflow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_multiplication_overflow_SOURCES = test/fuzz/multiplication_overflow.cpp - -test_fuzz_net_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_net_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_net_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_net_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_net_SOURCES = test/fuzz/net.cpp - -test_fuzz_net_permissions_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_net_permissions_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_net_permissions_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_net_permissions_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_net_permissions_SOURCES = test/fuzz/net_permissions.cpp - -test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 -test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_netaddr_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_netaddr_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_netaddress_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_netaddress_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_netaddress_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_netaddress_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_netaddress_SOURCES = test/fuzz/netaddress.cpp - -test_fuzz_out_point_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DOUT_POINT_DESERIALIZE=1 -test_fuzz_out_point_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_out_point_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_out_point_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_out_point_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_p2p_transport_deserializer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_p2p_transport_deserializer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_p2p_transport_deserializer_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_p2p_transport_deserializer_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_p2p_transport_deserializer_SOURCES = test/fuzz/p2p_transport_deserializer.cpp - -test_fuzz_parse_hd_keypath_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_hd_keypath_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_hd_keypath_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_hd_keypath_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_parse_hd_keypath_SOURCES = test/fuzz/parse_hd_keypath.cpp - -test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_iso8601_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_parse_iso8601_SOURCES = test/fuzz/parse_iso8601.cpp - -test_fuzz_parse_numbers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_numbers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_numbers_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_numbers_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_parse_numbers_SOURCES = test/fuzz/parse_numbers.cpp - -test_fuzz_parse_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_parse_script_SOURCES = test/fuzz/parse_script.cpp - -test_fuzz_parse_univalue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_univalue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_univalue_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_parse_univalue_SOURCES = test/fuzz/parse_univalue.cpp - -test_fuzz_prevector_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_prevector_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_prevector_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_prevector_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_prevector_SOURCES = test/fuzz/prevector.cpp - -test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 -test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_partial_merkle_tree_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 -test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_partially_signed_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_policy_estimator_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_policy_estimator_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_policy_estimator_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_policy_estimator_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_policy_estimator_SOURCES = test/fuzz/policy_estimator.cpp - -test_fuzz_policy_estimator_io_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_policy_estimator_io_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_policy_estimator_io_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_policy_estimator_io_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_policy_estimator_io_SOURCES = test/fuzz/policy_estimator_io.cpp - -test_fuzz_pow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_pow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_pow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_pow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_pow_SOURCES = test/fuzz/pow.cpp - -test_fuzz_prefilled_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPREFILLED_TRANSACTION_DESERIALIZE=1 -test_fuzz_prefilled_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_primitives_transaction_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp - -test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_messages_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_messages_SOURCES = test/fuzz/process_messages.cpp - -test_fuzz_process_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_process_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_addr_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=addr -test_fuzz_process_message_addr_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_addr_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_addr_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_addr_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=block -test_fuzz_process_message_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_block_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_block_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_block_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_blocktxn_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=blocktxn -test_fuzz_process_message_blocktxn_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_blocktxn_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_blocktxn_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_blocktxn_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_cmpctblock_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=cmpctblock -test_fuzz_process_message_cmpctblock_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_cmpctblock_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_cmpctblock_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_cmpctblock_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_feefilter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=feefilter -test_fuzz_process_message_feefilter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_feefilter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_feefilter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_feefilter_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_filteradd_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filteradd -test_fuzz_process_message_filteradd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_filteradd_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filteradd_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_filteradd_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_filterclear_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filterclear -test_fuzz_process_message_filterclear_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_filterclear_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filterclear_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_filterclear_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_filterload_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filterload -test_fuzz_process_message_filterload_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_filterload_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filterload_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_filterload_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_getaddr_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getaddr -test_fuzz_process_message_getaddr_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_getaddr_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getaddr_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_getaddr_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_getblocks_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getblocks -test_fuzz_process_message_getblocks_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_getblocks_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getblocks_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_getblocks_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_getblocktxn_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getblocktxn -test_fuzz_process_message_getblocktxn_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_getblocktxn_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getblocktxn_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_getblocktxn_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_getdata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getdata -test_fuzz_process_message_getdata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_getdata_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getdata_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_getdata_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_getheaders_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getheaders -test_fuzz_process_message_getheaders_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_getheaders_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getheaders_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_getheaders_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_headers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=headers -test_fuzz_process_message_headers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_headers_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_headers_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_headers_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_inv_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=inv -test_fuzz_process_message_inv_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_inv_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_inv_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_inv_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_mempool_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=mempool -test_fuzz_process_message_mempool_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_mempool_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_mempool_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_mempool_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_notfound_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=notfound -test_fuzz_process_message_notfound_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_notfound_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_notfound_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_notfound_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_ping_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=ping -test_fuzz_process_message_ping_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_ping_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_ping_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_ping_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_pong_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=pong -test_fuzz_process_message_pong_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_pong_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_pong_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_pong_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_sendcmpct_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=sendcmpct -test_fuzz_process_message_sendcmpct_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_sendcmpct_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_sendcmpct_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_sendcmpct_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_sendheaders_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=sendheaders -test_fuzz_process_message_sendheaders_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_sendheaders_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_sendheaders_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_sendheaders_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=tx -test_fuzz_process_message_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_tx_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_tx_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_verack_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=verack -test_fuzz_process_message_verack_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_verack_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_verack_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_verack_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_process_message_version_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=version -test_fuzz_process_message_version_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_process_message_version_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_version_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_process_message_version_SOURCES = test/fuzz/process_message.cpp - -test_fuzz_protocol_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_protocol_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_protocol_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_protocol_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_protocol_SOURCES = test/fuzz/protocol.cpp - -test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_psbt_SOURCES = test/fuzz/psbt.cpp - -test_fuzz_psbt_input_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_INPUT_DESERIALIZE=1 -test_fuzz_psbt_input_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_input_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_input_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_psbt_input_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_psbt_output_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_OUTPUT_DESERIALIZE=1 -test_fuzz_psbt_output_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_output_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_output_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_psbt_output_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_pub_key_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPUB_KEY_DESERIALIZE=1 -test_fuzz_pub_key_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_pub_key_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_pub_key_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_pub_key_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_random_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_random_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_random_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_random_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_random_SOURCES = test/fuzz/random.cpp - -test_fuzz_rbf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_rbf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_rbf_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_rbf_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_rbf_SOURCES = test/fuzz/rbf.cpp - -test_fuzz_rolling_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_rolling_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_rolling_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_rolling_bloom_filter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_rolling_bloom_filter_SOURCES = test/fuzz/rolling_bloom_filter.cpp - -test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_SOURCES = test/fuzz/script.cpp - -test_fuzz_script_bitcoin_consensus_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_bitcoin_consensus_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_bitcoin_consensus_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_bitcoin_consensus_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_bitcoin_consensus_SOURCES = test/fuzz/script_bitcoin_consensus.cpp - -test_fuzz_script_descriptor_cache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_descriptor_cache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_descriptor_cache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_descriptor_cache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_descriptor_cache_SOURCES = test/fuzz/script_descriptor_cache.cpp - -test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 -test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_flags_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_flags_SOURCES = test/fuzz/script_flags.cpp - -test_fuzz_script_interpreter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_interpreter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_interpreter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_interpreter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_interpreter_SOURCES = test/fuzz/script_interpreter.cpp - -test_fuzz_script_assets_test_minimizer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_assets_test_minimizer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_assets_test_minimizer_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_assets_test_minimizer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_script_assets_test_minimizer_SOURCES = test/fuzz/script_assets_test_minimizer.cpp - -test_fuzz_script_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_ops_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_ops_SOURCES = test/fuzz/script_ops.cpp - -test_fuzz_script_sigcache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_sigcache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_sigcache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_sigcache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_sigcache_SOURCES = test/fuzz/script_sigcache.cpp - -test_fuzz_script_sign_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_sign_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_sign_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_sign_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_script_sign_SOURCES = test/fuzz/script_sign.cpp - -test_fuzz_scriptnum_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_scriptnum_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_scriptnum_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_scriptnum_ops_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_scriptnum_ops_SOURCES = test/fuzz/scriptnum_ops.cpp - -test_fuzz_secp256k1_ec_seckey_import_export_der_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_secp256k1_ec_seckey_import_export_der_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_secp256k1_ec_seckey_import_export_der_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_secp256k1_ec_seckey_import_export_der_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_secp256k1_ec_seckey_import_export_der_SOURCES = test/fuzz/secp256k1_ec_seckey_import_export_der.cpp - -test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_SOURCES = test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp - -test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 -test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_service_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_service_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_signature_checker_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_signature_checker_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_signature_checker_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_signature_checker_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_signature_checker_SOURCES = test/fuzz/signature_checker.cpp - -test_fuzz_signet_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_signet_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_signet_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_signet_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_signet_SOURCES = test/fuzz/signet.cpp - -test_fuzz_snapshotmetadata_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSNAPSHOTMETADATA_DESERIALIZE=1 -test_fuzz_snapshotmetadata_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_snapshotmetadata_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_snapshotmetadata_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_snapshotmetadata_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_span_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_span_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_span_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_span_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_span_SOURCES = test/fuzz/span.cpp - -test_fuzz_spanparsing_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_spanparsing_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_spanparsing_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_spanparsing_SOURCES = test/fuzz/spanparsing.cpp - -test_fuzz_string_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_string_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_string_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_string_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_string_SOURCES = test/fuzz/string.cpp - -test_fuzz_strprintf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_strprintf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_strprintf_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_strprintf_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_strprintf_SOURCES = test/fuzz/strprintf.cpp - -test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 -test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_sub_net_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_sub_net_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_system_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_system_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_system_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_system_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_system_SOURCES = test/fuzz/system.cpp - -test_fuzz_timedata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_timedata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_timedata_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_timedata_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_timedata_SOURCES = test/fuzz/timedata.cpp - -test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_transaction_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_transaction_SOURCES = test/fuzz/transaction.cpp - -test_fuzz_tx_in_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_tx_in_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_in_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_in_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_tx_in_SOURCES = test/fuzz/tx_in.cpp - -test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 -test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_in_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_tx_in_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_tx_out_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_tx_out_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_out_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_out_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_tx_out_SOURCES = test/fuzz/tx_out.cpp - -test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 -test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_txoutcompressor_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_txrequest_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_txrequest_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txrequest_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txrequest_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_txrequest_SOURCES = test/fuzz/txrequest.cpp - -test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 -test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txundo_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_txundo_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_uint160_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DUINT160_DESERIALIZE=1 -test_fuzz_uint160_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_uint160_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_uint160_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_uint160_deserialize_SOURCES = test/fuzz/deserialize.cpp - -test_fuzz_uint256_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DUINT256_DESERIALIZE=1 -test_fuzz_uint256_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_uint256_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_uint256_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) -test_fuzz_uint256_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_fuzz_SOURCES = \ + test/fuzz/addition_overflow.cpp \ + test/fuzz/addrdb.cpp \ + test/fuzz/addrman.cpp \ + test/fuzz/asmap.cpp \ + test/fuzz/asmap_direct.cpp \ + test/fuzz/autofile.cpp \ + test/fuzz/banman.cpp \ + test/fuzz/base_encode_decode.cpp \ + test/fuzz/bech32.cpp \ + test/fuzz/block.cpp \ + test/fuzz/block_header.cpp \ + test/fuzz/blockfilter.cpp \ + test/fuzz/bloom_filter.cpp \ + test/fuzz/buffered_file.cpp \ + test/fuzz/chain.cpp \ + test/fuzz/checkqueue.cpp \ + test/fuzz/coins_view.cpp \ + test/fuzz/connman.cpp \ + test/fuzz/crypto.cpp \ + test/fuzz/crypto_aes256.cpp \ + test/fuzz/crypto_aes256cbc.cpp \ + test/fuzz/crypto_chacha20.cpp \ + test/fuzz/crypto_chacha20_poly1305_aead.cpp \ + test/fuzz/crypto_common.cpp \ + test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ + test/fuzz/crypto_poly1305.cpp \ + test/fuzz/cuckoocache.cpp \ + test/fuzz/data_stream.cpp \ + test/fuzz/decode_tx.cpp \ + test/fuzz/descriptor_parse.cpp \ + test/fuzz/deserialize.cpp \ + test/fuzz/eval_script.cpp \ + test/fuzz/fee_rate.cpp \ + test/fuzz/fees.cpp \ + test/fuzz/flatfile.cpp \ + test/fuzz/float.cpp \ + test/fuzz/golomb_rice.cpp \ + test/fuzz/hex.cpp \ + test/fuzz/http_request.cpp \ + test/fuzz/integer.cpp \ + test/fuzz/key.cpp \ + test/fuzz/key_io.cpp \ + test/fuzz/kitchen_sink.cpp \ + test/fuzz/load_external_block_file.cpp \ + test/fuzz/locale.cpp \ + test/fuzz/merkleblock.cpp \ + test/fuzz/message.cpp \ + test/fuzz/muhash.cpp \ + test/fuzz/multiplication_overflow.cpp \ + test/fuzz/net.cpp \ + test/fuzz/net_permissions.cpp \ + test/fuzz/netaddress.cpp \ + test/fuzz/node_eviction.cpp \ + test/fuzz/p2p_transport_deserializer.cpp \ + test/fuzz/parse_hd_keypath.cpp \ + test/fuzz/parse_iso8601.cpp \ + test/fuzz/parse_numbers.cpp \ + test/fuzz/parse_script.cpp \ + test/fuzz/parse_univalue.cpp \ + test/fuzz/policy_estimator.cpp \ + test/fuzz/policy_estimator_io.cpp \ + test/fuzz/pow.cpp \ + test/fuzz/prevector.cpp \ + test/fuzz/primitives_transaction.cpp \ + test/fuzz/process_message.cpp \ + test/fuzz/process_messages.cpp \ + test/fuzz/protocol.cpp \ + test/fuzz/psbt.cpp \ + test/fuzz/random.cpp \ + test/fuzz/rbf.cpp \ + test/fuzz/rolling_bloom_filter.cpp \ + test/fuzz/script.cpp \ + test/fuzz/script_assets_test_minimizer.cpp \ + test/fuzz/script_bitcoin_consensus.cpp \ + test/fuzz/script_descriptor_cache.cpp \ + test/fuzz/script_flags.cpp \ + test/fuzz/script_interpreter.cpp \ + test/fuzz/script_ops.cpp \ + test/fuzz/script_sigcache.cpp \ + test/fuzz/script_sign.cpp \ + test/fuzz/scriptnum_ops.cpp \ + test/fuzz/secp256k1_ec_seckey_import_export_der.cpp \ + test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp \ + test/fuzz/signature_checker.cpp \ + test/fuzz/signet.cpp \ + test/fuzz/span.cpp \ + test/fuzz/spanparsing.cpp \ + test/fuzz/string.cpp \ + test/fuzz/strprintf.cpp \ + test/fuzz/system.cpp \ + test/fuzz/timedata.cpp \ + test/fuzz/transaction.cpp \ + test/fuzz/tx_in.cpp \ + test/fuzz/tx_out.cpp \ + test/fuzz/txrequest.cpp endif # ENABLE_FUZZ @@ -1305,6 +324,11 @@ if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check endif +if ENABLE_FUZZ_LINK_ALL +all-local: $(FUZZ_BINARY) + bash ./test/fuzz/danger_link_all.sh +endif + %.cpp.test: %.cpp @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 27f22826a9..8f77ed35ce 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/addrdb.h b/src/addrdb.h index 4ac0e3e1b5..8953ebb169 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/addrman.cpp b/src/addrman.cpp index 7636c6bad2..ed7fccc0ff 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -617,7 +617,7 @@ CAddrInfo CAddrMan::SelectTriedCollision_() return CAddrInfo(); } - CAddrInfo& newInfo = mapInfo[id_new]; + const CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); diff --git a/src/addrman.h b/src/addrman.h index 04dd30b375..9ac67b7af6 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -281,8 +281,18 @@ protected: //! Select several addresses at once. void GetAddr_(std::vector<CAddress> &vAddr, size_t max_addresses, size_t max_pct) EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Mark an entry as currently-connected-to. - void Connected_(const CService &addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + /** We have successfully connected to this peer. Calling this function + * updates the CAddress's nTime, which is used in our IsTerrible() + * decisions and gossiped to peers. Callers should be careful that updating + * this information doesn't leak topology information to network spies. + * + * net_processing calls this function when it *disconnects* from a peer to + * not leak information about currently connected peers. + * + * @param[in] addr The address of the peer we were connected to + * @param[in] nTime The time that we were last connected to this peer + */ + void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Update an entry's service bits. void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -704,7 +714,7 @@ public: return vAddr; } - //! Mark an entry as currently-connected-to. + //! Outer function for Connected_() void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) { LOCK(cs); diff --git a/src/attributes.h b/src/attributes.h index 45099bd8b8..9957bcd84b 100644 --- a/src/attributes.h +++ b/src/attributes.h @@ -1,22 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ATTRIBUTES_H #define BITCOIN_ATTRIBUTES_H -#if defined(__has_cpp_attribute) -# if __has_cpp_attribute(nodiscard) -# define NODISCARD [[nodiscard]] -# endif -#endif -#ifndef NODISCARD -# if defined(_MSC_VER) && _MSC_VER >= 1700 -# define NODISCARD _Check_return_ +#if defined(__clang__) +# if __has_attribute(lifetimebound) +# define LIFETIMEBOUND [[clang::lifetimebound]] # else -# define NODISCARD __attribute__((warn_unused_result)) +# define LIFETIMEBOUND # endif +#else +# define LIFETIMEBOUND #endif #endif // BITCOIN_ATTRIBUTES_H diff --git a/src/banman.cpp b/src/banman.cpp index 995fef3d07..49bf6c43dc 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/base58.cpp b/src/base58.cpp index 0dc6044145..65e373283c 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,7 +35,7 @@ static const int8_t mapBase58[256] = { -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, }; -NODISCARD static bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_len) +[[nodiscard]] static bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_len) { // Skip leading spaces. while (*psz && IsSpace(*psz)) @@ -141,7 +141,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input) return EncodeBase58(vch); } -NODISCARD static bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len) +[[nodiscard]] static bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len) { if (!DecodeBase58(psz, vchRet, max_ret_len > std::numeric_limits<int>::max() - 4 ? std::numeric_limits<int>::max() : max_ret_len + 4) || (vchRet.size() < 4)) { diff --git a/src/base58.h b/src/base58.h index 468c3e2589..9ba5af73e0 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,7 +29,7 @@ std::string EncodeBase58(Span<const unsigned char> input); * Decode a base58-encoded string (str) into a byte vector (vchRet). * return true if decoding is successful. */ -NODISCARD bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); +[[nodiscard]] bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); /** * Encode a byte span into a base58-encoded string, including checksum @@ -40,6 +40,6 @@ std::string EncodeBase58Check(Span<const unsigned char> input); * Decode a base58-encoded string (str) that includes a checksum into a byte * vector (vchRet), return true if decoding is successful */ -NODISCARD bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); +[[nodiscard]] bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); #endif // BITCOIN_BASE58_H diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 18cb5de196..6f6b4e3bfa 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 99a7ad237b..af5a82f69f 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index 913e0f8d57..a6f4eec4ca 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index 3b1d3e697a..e994279a4d 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -31,12 +31,15 @@ static void CHACHA20_POLY1305_AEAD(benchmark::Bench& bench, size_t buffersize, b uint32_t len = 0; bench.batch(buffersize).unit("byte").run([&] { // encrypt or decrypt the buffer with a static key - assert(aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true)); + const bool crypt_ok_1 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); + assert(crypt_ok_1); if (include_decryption) { // if we decrypt, include the GetLength - assert(aead.GetLength(&len, seqnr_aad, aad_pos, in.data())); - assert(aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true)); + const bool get_length_ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); + assert(get_length_ok); + const bool crypt_ok_2 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); + assert(crypt_ok_2); } // increase main sequence number diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 99aafd8dfc..e50c4476bb 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 65d16d47d8..30fe11be6b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> +#include <crypto/muhash.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -105,6 +106,54 @@ static void FastRandom_1bit(benchmark::Bench& bench) }); } +static void MuHash(benchmark::Bench& bench) +{ + MuHash3072 acc; + unsigned char key[32] = {0}; + int i = 0; + bench.run([&] { + key[0] = ++i; + acc *= MuHash3072(key); + }); +} + +static void MuHashMul(benchmark::Bench& bench) +{ + MuHash3072 acc; + FastRandomContext rng(true); + MuHash3072 muhash{rng.randbytes(32)}; + + bench.run([&] { + acc *= muhash; + }); +} + +static void MuHashDiv(benchmark::Bench& bench) +{ + MuHash3072 acc; + FastRandomContext rng(true); + MuHash3072 muhash{rng.randbytes(32)}; + + for (size_t i = 0; i < bench.epochIterations(); ++i) { + acc *= muhash; + } + + bench.run([&] { + acc /= muhash; + }); +} + +static void MuHashPrecompute(benchmark::Bench& bench) +{ + MuHash3072 acc; + FastRandomContext rng(true); + std::vector<unsigned char> key{rng.randbytes(32)}; + + bench.run([&] { + MuHash3072{key}; + }); +} + BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); @@ -116,3 +165,8 @@ BENCHMARK(SipHash_32b); BENCHMARK(SHA256D64_1024); BENCHMARK(FastRandom_32bit); BENCHMARK(FastRandom_1bit); + +BENCHMARK(MuHash); +BENCHMARK(MuHashMul); +BENCHMARK(MuHashDiv); +BENCHMARK(MuHashPrecompute); diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index ef83242e41..607e4392b7 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp index 309cae3723..753c8c2881 100644 --- a/src/bench/hashpadding.cpp +++ b/src/bench/hashpadding.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index 32b060a15a..b6d8824aba 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 1b9e428c9d..db9a5661fd 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 89233e390c..9b862b735c 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp index d8db99e7d4..cdef97c0ea 100644 --- a/src/bench/poly1305.cpp +++ b/src/bench/poly1305.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index a2dbefa54a..dcd0e10285 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,24 +9,16 @@ #include <bench/bench.h> -// GCC 4.8 is missing some C++11 type_traits, -// https://www.gnu.org/software/gcc/gcc-5/changes.html -#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 5 -#define IS_TRIVIALLY_CONSTRUCTIBLE std::has_trivial_default_constructor -#else -#define IS_TRIVIALLY_CONSTRUCTIBLE std::is_trivially_default_constructible -#endif - struct nontrivial_t { int x; nontrivial_t() :x(-1) {} SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); } }; -static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE<nontrivial_t>::value, +static_assert(!std::is_trivially_default_constructible<nontrivial_t>::value, "expected nontrivial_t to not be trivially constructible"); typedef unsigned char trivial_t; -static_assert(IS_TRIVIALLY_CONSTRUCTIBLE<trivial_t>::value, +static_assert(std::is_trivially_default_constructible<trivial_t>::value, "expected trivial_t to be trivially constructible"); template <typename T> diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 9b43951e6e..997ab56549 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 1ff41765cf..f1eeef8885 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index fad179eb87..afc733482e 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 9af0b502eb..e3f6b35a7d 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -24,7 +24,7 @@ static void VerifyScriptBench(benchmark::Bench& bench) const int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; const int witnessversion = 0; - // Keypair. + // Key pair. CKey key; static const std::array<unsigned char, 32> vchKey = { { diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index b3b73284d8..b385cec085 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -24,21 +24,19 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - NodeContext node; - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet{chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup.m_node.chain.get(), "", CreateMockWalletDatabase()}; { wallet.SetupLegacyScriptPubKeyMan(); bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); } - auto handler = chain->handleNotifications({&wallet, [](CWallet*) {}}); + auto handler = test_setup.m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); const Optional<std::string> address_mine{add_mine ? Optional<std::string>{getnewaddress(wallet)} : nullopt}; if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { - generatetoaddress(test_setup.m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(test_setup.m_node, address_mine.value_or(ADDRESS_WATCHONLY)); generatetoaddress(test_setup.m_node, ADDRESS_WATCHONLY); } SyncWithValidationInterfaceQueue(); diff --git a/src/bitcoin-cli-res.rc b/src/bitcoin-cli-res.rc index 58f8f1e8a2..405a302261 100644 --- a/src/bitcoin-cli-res.rc +++ b/src/bitcoin-cli-res.rc @@ -1,8 +1,8 @@ #include <windows.h> // needed for VERSIONINFO #include "clientversion.h" // holds the needed client version information -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION #define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index ef4641cb63..94043a6b45 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -59,7 +59,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0).", ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS); + argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); SetupChainParamsBaseOptions(argsman); argsman.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -309,7 +309,8 @@ private: } return UNKNOWN_NETWORK; } - uint8_t m_details_level{0}; //!< Optional user-supplied arg to set dashboard details level + uint8_t m_details_level{0}; //!< Optional user-supplied arg to set dashboard details level + bool m_is_help_requested{false}; //!< Optional user-supplied arg to print help documentation bool DetailsRequested() const { return m_details_level > 0 && m_details_level < 5; } bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } @@ -349,6 +350,62 @@ private: const double milliseconds{round(1000 * seconds)}; return milliseconds > 999999 ? "-" : ToString(milliseconds); } + const UniValue NetinfoHelp() + { + return std::string{ + "-netinfo level|\"help\" \n\n" + "Returns a network peer connections dashboard with information from the remote server.\n" + "Under the hood, -netinfo fetches the data by calling getpeerinfo and getnetworkinfo.\n" + "An optional integer argument from 0 to 4 can be passed for different peers listings.\n" + "Pass \"help\" to see this detailed help documentation.\n" + "If more than one argument is passed, only the first one is read and parsed.\n" + "Suggestion: use with the Linux watch(1) command for a live dashboard; see example below.\n\n" + "Arguments:\n" + "1. level (integer 0-4, optional) Specify the info level of the peers dashboard (default 0):\n" + " 0 - Connection counts and local addresses\n" + " 1 - Like 0 but with a peers listing (without address or version columns)\n" + " 2 - Like 1 but with an address column\n" + " 3 - Like 1 but with a version column\n" + " 4 - Like 1 but with both address and version columns\n" + "2. help (string \"help\", optional) Print this help documentation instead of the dashboard.\n\n" + "Result:\n\n" + "* The peers listing in levels 1-4 displays all of the peers sorted by direction and minimum ping time:\n\n" + " Column Description\n" + " ------ -----------\n" + " <-> Direction\n" + " \"in\" - inbound connections are those initiated by the peer\n" + " \"out\" - outbound connections are those initiated by us\n" + " type Type of peer connection\n" + " \"full\" - full relay, the default\n" + " \"block\" - block relay; like full relay but does not relay transactions or addresses\n" + " net Network the peer connected through (\"ipv4\", \"ipv6\", \"onion\", \"i2p\", or \"cjdns\")\n" + " mping Minimum observed ping time, in milliseconds (ms)\n" + " ping Last observed ping time, in milliseconds (ms)\n" + " send Time since last message sent to the peer, in seconds\n" + " recv Time since last message received from the peer, in seconds\n" + " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" + " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" + " age Duration of connection to the peer, in minutes\n" + " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" + " peer selection (only displayed if the -asmap config option is set)\n" + " id Peer index, in increasing order of peer connections since node startup\n" + " address IP address and port of the peer\n" + " version Peer version and subversion concatenated, e.g. \"70016/Satoshi:21.0.0/\"\n\n" + "* The connection counts table displays the number of peers by direction, network, and the totals\n" + " for each, as well as a column for block relay peers.\n\n" + "* The local addresses table lists each local address broadcast by the node, the port, and the score.\n\n" + "Examples:\n\n" + "Connection counts and local addresses only\n" + "> bitcoin-cli -netinfo\n\n" + "Compact peers listing\n" + "> bitcoin-cli -netinfo 1\n\n" + "Full dashboard\n" + "> bitcoin-cli -netinfo 4\n\n" + "Full live dashboard, adjust --interval or --no-title as needed (Linux)\n" + "> watch --interval 1 --no-title bitcoin-cli -netinfo 4\n\n" + "See this help\n" + "> bitcoin-cli -netinfo help\n"}; + } const int64_t m_time_now{GetSystemTimeInSeconds()}; public: @@ -361,6 +418,10 @@ public: uint8_t n{0}; if (ParseUInt8(args.at(0), &n)) { m_details_level = n; + } else if (args.at(0) == "help") { + m_is_help_requested = true; + } else { + throw std::runtime_error(strprintf("invalid -netinfo argument: %s", args.at(0))); } } UniValue result(UniValue::VARR); @@ -371,6 +432,9 @@ public: UniValue ProcessReply(const UniValue& batch_in) override { + if (m_is_help_requested) { + return JSONRPCReplyObj(NetinfoHelp(), NullUniValue, 1); + } const std::vector<UniValue> batch{JSONRPCProcessBatchReply(batch_in)}; if (!batch[ID_PEERINFO]["error"].isNull()) return batch[ID_PEERINFO]; if (!batch[ID_NETWORKINFO]["error"].isNull()) return batch[ID_NETWORKINFO]; @@ -424,7 +488,7 @@ public: // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("Peer connections sorted by direction and min ping\n<-> relay net mping ping send recv txn blk %*s ", m_max_age_length, "age"); + result += strprintf("<-> relay net mping ping send recv txn blk %*s ", m_max_age_length, "age"); if (m_is_asmap_on) result += " asmap "; result += strprintf("%*s %-*s%s\n", m_max_id_length, "id", IsAddressSelected() ? m_max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); for (const Peer& peer : m_peers) { diff --git a/src/bitcoin-tx-res.rc b/src/bitcoin-tx-res.rc index 3e49b820bc..b545ce9dbe 100644 --- a/src/bitcoin-tx-res.rc +++ b/src/bitcoin-tx-res.rc @@ -1,8 +1,8 @@ #include <windows.h> // needed for VERSIONINFO #include "clientversion.h" // holds the needed client version information -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION #define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e22b3766cf..321d62fe4d 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -11,6 +11,7 @@ #include <consensus/consensus.h> #include <core_io.h> #include <key_io.h> +#include <policy/policy.h> #include <policy/rbf.h> #include <primitives/transaction.h> #include <script/script.h> @@ -40,6 +41,7 @@ static void SetupBitcoinTxArgs(ArgsManager &argsman) { SetupHelpOptions(argsman); + argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-create", "Create new, empty TX.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-json", "Select JSON output", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-txid", "Output only the hex-encoded transaction id of the resultant transaction.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -95,13 +97,16 @@ static int AppInitRawTx(int argc, char* argv[]) fCreateBlank = gArgs.GetBoolArg("-create", false); - if (argc < 2 || HelpRequested(gArgs)) { + if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { // First part of help message is specific to this utility - std::string strUsage = PACKAGE_NAME " bitcoin-tx utility version " + FormatFullVersion() + "\n\n" + - "Usage: bitcoin-tx [options] <hex-tx> [commands] Update hex-encoded bitcoin transaction\n" + - "or: bitcoin-tx [options] -create [commands] Create hex-encoded bitcoin transaction\n" + - "\n"; - strUsage += gArgs.GetHelpMessage(); + std::string strUsage = PACKAGE_NAME " bitcoin-tx utility version " + FormatFullVersion() + "\n"; + if (!gArgs.IsArgSet("-version")) { + strUsage += "\n" + "Usage: bitcoin-tx [options] <hex-tx> [commands] Update hex-encoded bitcoin transaction\n" + "or: bitcoin-tx [options] -create [commands] Create hex-encoded bitcoin transaction\n" + "\n"; + strUsage += gArgs.GetHelpMessage(); + } tfm::format(std::cout, "%s", strUsage); @@ -192,8 +197,9 @@ static CAmount ExtractAndValidateValue(const std::string& strValue) static void MutateTxVersion(CMutableTransaction& tx, const std::string& cmdVal) { int64_t newVersion; - if (!ParseInt64(cmdVal, &newVersion) || newVersion < 1 || newVersion > CTransaction::MAX_STANDARD_VERSION) + if (!ParseInt64(cmdVal, &newVersion) || newVersion < 1 || newVersion > TX_MAX_STANDARD_VERSION) { throw std::runtime_error("Invalid TX version requested: '" + cmdVal + "'"); + } tx.nVersion = (int) newVersion; } diff --git a/src/bitcoin-util-res.rc b/src/bitcoin-util-res.rc new file mode 100644 index 0000000000..3f0fa8ab6d --- /dev/null +++ b/src/bitcoin-util-res.rc @@ -0,0 +1,35 @@ +#include <windows.h> // needed for VERSIONINFO +#include "clientversion.h" // holds the needed client version information + +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_FILEVERSION VER_PRODUCTVERSION +#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR + +VS_VERSION_INFO VERSIONINFO +FILEVERSION VER_FILEVERSION +PRODUCTVERSION VER_PRODUCTVERSION +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_APP +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" // U.S. English - multilingual (hex) + BEGIN + VALUE "CompanyName", "Bitcoin" + VALUE "FileDescription", "bitcoin-util (CLI Bitcoin utility)" + VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "InternalName", "bitcoin-util" + VALUE "LegalCopyright", COPYRIGHT_STR + VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." + VALUE "OriginalFilename", "bitcoin-util.exe" + VALUE "ProductName", "bitcoin-util" + VALUE "ProductVersion", VER_PRODUCTVERSION_STR + END + END + + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0, 1252 // language neutral - multilingual (decimal) + END +END diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp new file mode 100644 index 0000000000..b702a68bdf --- /dev/null +++ b/src/bitcoin-util.cpp @@ -0,0 +1,211 @@ +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <arith_uint256.h> +#include <clientversion.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <core_io.h> +#include <key_io.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <univalue.h> +#include <util/moneystr.h> +#include <util/rbf.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/system.h> +#include <util/translation.h> + +#include <functional> +#include <memory> +#include <stdio.h> +#include <thread> + +#include <boost/algorithm/string.hpp> + +static const int CONTINUE_EXECUTION=-1; + +const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; + +static void SetupBitcoinUtilArgs(ArgsManager &argsman) +{ + SetupHelpOptions(argsman); + + argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + + SetupChainParamsBaseOptions(argsman); +} + +// This function returns either one of EXIT_ codes when it's expected to stop the process or +// CONTINUE_EXECUTION when it's expected to continue further. +static int AppInitUtil(int argc, char* argv[]) +{ + SetupBitcoinUtilArgs(gArgs); + std::string error; + if (!gArgs.ParseParameters(argc, argv, error)) { + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); + return EXIT_FAILURE; + } + + // Check for chain settings (Params() calls are only valid after this clause) + try { + SelectParams(gArgs.GetChainName()); + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } + + if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + // First part of help message is specific to this utility + std::string strUsage = PACKAGE_NAME " bitcoin-util utility version " + FormatFullVersion() + "\n"; + if (!gArgs.IsArgSet("-version")) { + strUsage += "\n" + "Usage: bitcoin-util [options] [commands] Do stuff\n"; + strUsage += "\n" + gArgs.GetHelpMessage(); + } + + tfm::format(std::cout, "%s", strUsage); + + if (argc < 2) { + tfm::format(std::cerr, "Error: too few parameters\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; + } + return CONTINUE_EXECUTION; +} + +static void grind_task(uint32_t nBits, CBlockHeader& header_orig, uint32_t offset, uint32_t step, std::atomic<bool>& found) +{ + arith_uint256 target; + bool neg, over; + target.SetCompact(nBits, &neg, &over); + if (target == 0 || neg || over) return; + CBlockHeader header = header_orig; // working copy + header.nNonce = offset; + + uint32_t finish = std::numeric_limits<uint32_t>::max() - step; + finish = finish - (finish % step) + offset; + + while (!found && header.nNonce < finish) { + const uint32_t next = (finish - header.nNonce < 5000*step) ? finish : header.nNonce + 5000*step; + do { + if (UintToArith256(header.GetHash()) <= target) { + if (!found.exchange(true)) { + header_orig.nNonce = header.nNonce; + } + return; + } + header.nNonce += step; + } while(header.nNonce != next); + } +} + +static int Grind(int argc, char* argv[], std::string& strPrint) +{ + if (argc != 1) { + strPrint = "Must specify block header to grind"; + return 1; + } + + CBlockHeader header; + if (!DecodeHexBlockHeader(header, argv[0])) { + strPrint = "Could not decode block header"; + return 1; + } + + uint32_t nBits = header.nBits; + std::atomic<bool> found{false}; + + std::vector<std::thread> threads; + int n_tasks = std::max(1u, std::thread::hardware_concurrency()); + for (int i = 0; i < n_tasks; ++i) { + threads.emplace_back( grind_task, nBits, std::ref(header), i, n_tasks, std::ref(found) ); + } + for (auto& t : threads) { + t.join(); + } + if (!found) { + strPrint = "Could not satisfy difficulty target"; + return 1; + } + + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << header; + strPrint = HexStr(ss); + return 0; +} + +static int CommandLineUtil(int argc, char* argv[]) +{ + if (argc <= 1) return 1; + + std::string strPrint; + int nRet = 0; + + try { + while (argc > 1 && IsSwitchChar(argv[1][0]) && (argv[1][1] != 0)) { + --argc; + ++argv; + } + + char* command = argv[1]; + if (strcmp(command, "grind") == 0) { + nRet = Grind(argc-2, argv+2, strPrint); + } else { + strPrint = strprintf("Unknown command %s", command); + nRet = 1; + } + } + catch (const std::exception& e) { + strPrint = std::string("error: ") + e.what(); + nRet = EXIT_FAILURE; + } + catch (...) { + PrintExceptionContinue(nullptr, "CommandLineUtil()"); + throw; + } + + if (strPrint != "") { + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); + } + return nRet; +} + +int main(int argc, char* argv[]) +{ + SetupEnvironment(); + + try { + int ret = AppInitUtil(argc, argv); + if (ret != CONTINUE_EXECUTION) + return ret; + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "AppInitUtil()"); + return EXIT_FAILURE; + } catch (...) { + PrintExceptionContinue(nullptr, "AppInitUtil()"); + return EXIT_FAILURE; + } + + int ret = EXIT_FAILURE; + try { + ret = CommandLineUtil(argc, argv); + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "CommandLineUtil()"); + } catch (...) { + PrintExceptionContinue(nullptr, "CommandLineUtil()"); + } + return ret; +} diff --git a/src/bitcoin-wallet-res.rc b/src/bitcoin-wallet-res.rc index e9fa2dbb40..59346ab8f6 100644 --- a/src/bitcoin-wallet-res.rc +++ b/src/bitcoin-wallet-res.rc @@ -1,8 +1,8 @@ #include <windows.h> // needed for VERSIONINFO #include "clientversion.h" // holds the needed client version information -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION #define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index b9c2fe2d34..3e8e5fc7bc 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -24,14 +24,20 @@ static void SetupWalletToolArgs(ArgsManager& argsman) SetupHelpOptions(argsman); SetupChainParamsBaseOptions(argsman); + argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-wallet=<wallet-name>", "Specify wallet name", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::OPTIONS); + argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); } static bool WalletAppInit(int argc, char* argv[]) @@ -42,16 +48,18 @@ static bool WalletAppInit(int argc, char* argv[]) tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } - if (argc < 2 || HelpRequested(gArgs)) { - std::string usage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n\n" + - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + - "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + - "Usage:\n" + - " bitcoin-wallet [options] <command>\n\n" + - gArgs.GetHelpMessage(); - - tfm::format(std::cout, "%s", usage); + if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; + if (!gArgs.IsArgSet("-version")) { + strUsage += "\n" + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n"; + strUsage += "\n" + gArgs.GetHelpMessage(); + } + tfm::format(std::cout, "%s", strUsage); return false; } @@ -112,8 +120,9 @@ int main(int argc, char* argv[]) ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(method, name)) + if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) { return EXIT_FAILURE; + } ECC_Stop(); return EXIT_SUCCESS; } diff --git a/src/bitcoind-res.rc b/src/bitcoind-res.rc index 3a64acd5d1..a98b50c899 100644 --- a/src/bitcoind-res.rc +++ b/src/bitcoind-res.rc @@ -1,8 +1,8 @@ #include <windows.h> // needed for VERSIONINFO #include "clientversion.h" // holds the needed client version information -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION #define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 455a82e390..b7bcb534ef 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -28,15 +28,6 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; UrlDecodeFn* const URL_DECODE = urlDecode; -static void WaitForShutdown(NodeContext& node) -{ - while (!ShutdownRequested()) - { - UninterruptibleSleep(std::chrono::milliseconds{200}); - } - Interrupt(node); -} - static bool AppInit(int argc, char* argv[]) { NodeContext node; @@ -57,11 +48,11 @@ static bool AppInit(int argc, char* argv[]) if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; - if (args.IsArgSet("-version")) { - strUsage += FormatParagraph(LicenseInfo()) + "\n"; - } else { - strUsage += "\nUsage: bitcoind [options] Start " PACKAGE_NAME "\n"; - strUsage += "\n" + args.GetHelpMessage(); + if (!args.IsArgSet("-version")) { + strUsage += FormatParagraph(LicenseInfo()) + "\n" + "\nUsage: bitcoind [options] Start " PACKAGE_NAME "\n" + "\n"; + strUsage += args.GetHelpMessage(); } tfm::format(std::cout, "%s", strUsage); @@ -147,12 +138,10 @@ static bool AppInit(int argc, char* argv[]) PrintExceptionContinue(nullptr, "AppInit()"); } - if (!fRet) - { - Interrupt(node); - } else { - WaitForShutdown(node); + if (fRet) { + WaitForShutdown(); } + Interrupt(node); Shutdown(node); return fRet; diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index a47709cd82..aa111b5939 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 9a6fb4abd0..41fa0b6fa0 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.cpp b/src/bloom.cpp index d182f0728e..d0128a26d7 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.h b/src/bloom.h index 9307257852..fdaa8abfb2 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -94,7 +94,18 @@ public: * insert()'ed ... but may also return true for items that were not inserted. * * It needs around 1.8 bytes per element per factor 0.1 of false positive rate. - * (More accurately: 3/(log(256)*log(2)) * log(1/fpRate) * nElements bytes) + * For example, if we want 1000 elements, we'd need: + * - ~1800 bytes for a false positive rate of 0.1 + * - ~3600 bytes for a false positive rate of 0.01 + * - ~5400 bytes for a false positive rate of 0.001 + * + * If we make these simplifying assumptions: + * - logFpRate / log(0.5) doesn't get rounded or clamped in the nHashFuncs calculation + * - nElements is even, so that nEntriesPerGeneration == nElements / 2 + * + * Then we get a more accurate estimate for filter bytes: + * + * 3/(log(256)*log(2)) * log(1/fpRate) * nElements */ class CRollingBloomFilter { diff --git a/src/chainparams.cpp b/src/chainparams.cpp index fedb032db2..88cf5ef0a8 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -272,7 +272,7 @@ public: bin = ParseHex("512103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae"); vSeeds.emplace_back("178.128.221.177"); vSeeds.emplace_back("2a01:7c8:d005:390::5"); - vSeeds.emplace_back("ntv3mtqw5wt63red.onion:38333"); + vSeeds.emplace_back("v7ajjeirttkbnt32wpy3c6w3emwnfr3fkla7hpxcfokr3ysd3kqtzmqd.onion:38333"); consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000000000000000000019fd16269a"); consensus.defaultAssumeValid = uint256S("0x0000002a1de0f46379358c1fd09906f7ac59adf3712323ed90eb59e4c183c020"); // 9434 diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index 603969aaea..2c517b58f8 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 9b4ae2f7ab..d593cff722 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 993967a180..29c38e2d3b 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,8 +30,7 @@ const std::string CLIENT_NAME("Satoshi"); #define BUILD_DESC BUILD_GIT_TAG #define BUILD_SUFFIX "" #else - #define BUILD_DESC "v" STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) \ - "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) + #define BUILD_DESC "v" STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #ifdef BUILD_GIT_COMMIT #define BUILD_SUFFIX "-" BUILD_GIT_COMMIT #elif defined(GIT_COMMIT_ID) @@ -45,10 +44,7 @@ const std::string CLIENT_BUILD(BUILD_DESC BUILD_SUFFIX); static std::string FormatVersion(int nVersion) { - if (nVersion % 100 == 0) - return strprintf("%d.%d.%d", nVersion / 1000000, (nVersion / 10000) % 100, (nVersion / 100) % 100); - else - return strprintf("%d.%d.%d.%d", nVersion / 1000000, (nVersion / 10000) % 100, (nVersion / 100) % 100, nVersion % 100); + return strprintf("%d.%d.%d", nVersion / 10000, (nVersion / 100) % 100, nVersion % 100); } std::string FormatFullVersion() diff --git a/src/clientversion.h b/src/clientversion.h index 363094b696..2da909f829 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ #endif //HAVE_CONFIG_H // Check that required client information is defined -#if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_REVISION) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) +#if !defined(CLIENT_VERSION_MAJOR) || !defined(CLIENT_VERSION_MINOR) || !defined(CLIENT_VERSION_BUILD) || !defined(CLIENT_VERSION_IS_RELEASE) || !defined(COPYRIGHT_YEAR) #error Client version information missing: version is not defined by bitcoin-config.h or in any other way #endif @@ -36,9 +36,8 @@ #include <vector> static const int CLIENT_VERSION = - 1000000 * CLIENT_VERSION_MAJOR - + 10000 * CLIENT_VERSION_MINOR - + 100 * CLIENT_VERSION_REVISION + 10000 * CLIENT_VERSION_MAJOR + + 100 * CLIENT_VERSION_MINOR + 1 * CLIENT_VERSION_BUILD; extern const std::string CLIENT_NAME; diff --git a/src/coins.cpp b/src/coins.cpp index 5de2ed7810..dd84e720e7 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,8 +31,6 @@ bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} - CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { diff --git a/src/coins.h b/src/coins.h index a3e241ac90..d2eb42d8cf 100644 --- a/src/coins.h +++ b/src/coins.h @@ -8,11 +8,11 @@ #include <compressor.h> #include <core_memusage.h> -#include <crypto/siphash.h> #include <memusage.h> #include <primitives/transaction.h> #include <serialize.h> #include <uint256.h> +#include <util/hasher.h> #include <assert.h> #include <stdint.h> @@ -82,33 +82,6 @@ public: } }; -class SaltedOutpointHasher -{ -private: - /** Salt */ - const uint64_t k0, k1; - -public: - SaltedOutpointHasher(); - - /** - * This *must* return size_t. With Boost 1.46 on 32-bit systems the - * unordered_map will behave unpredictably if the custom hasher returns a - * uint64_t, resulting in failures when syncing the chain (#4634). - * - * Having the hash noexcept allows libstdc++'s unordered_map to recalculate - * the hash during rehash, so it does not have to cache the value. This - * reduces node's memory by sizeof(size_t). The required recalculation has - * a slight performance penalty (around 1.6%), but this is compensated by - * memory savings of about 9% which allow for a larger dbcache setting. - * - * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html - */ - size_t operator()(const COutPoint& id) const noexcept { - return SipHashUint256Extra(k0, k1, id.hash, id.n); - } -}; - /** * A Coin in one level of the coins database caching hierarchy. * diff --git a/src/compat.h b/src/compat.h index 0be02cae03..dad14748a2 100644 --- a/src/compat.h +++ b/src/compat.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,11 +18,7 @@ #undef FD_SETSIZE // prevent redefinition compiler warning #endif #define FD_SETSIZE 1024 // max number of fds in fd_set - -#include <winsock2.h> // Must be included before mswsock.h and windows.h - -#include <mswsock.h> -#include <windows.h> +#include <winsock2.h> #include <ws2tcpip.h> #include <stdint.h> #else diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 4b0b224c69..301c2d914c 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index d17de33e86..6d7a293f9b 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/glibc_sanity.cpp b/src/compat/glibc_sanity.cpp index 0367b9a53f..06d0dd6fba 100644 --- a/src/compat/glibc_sanity.cpp +++ b/src/compat/glibc_sanity.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/params.h b/src/consensus/params.h index 0983595c6a..217cb019e1 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 9e8e6530f1..f595f16eab 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index e2a9328df8..e78dc9f2a5 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/validation.h b/src/consensus/validation.h index e007c481df..c4d305434a 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_io.h b/src/core_io.h index 80ec80cd50..5469a760ee 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,18 +18,19 @@ class CTransaction; struct CMutableTransaction; class uint256; class UniValue; +class CTxUndo; // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); -NODISCARD bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); -NODISCARD bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); +[[nodiscard]] bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); +[[nodiscard]] bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); /** * Parse a hex string into 256 bits * @param[in] strHex a hex-formatted, 64-character string - * @param[out] result the result of the parasing + * @param[out] result the result of the parsing * @returns true if successful, false if not * * @see ParseHashV for an RPC-oriented version of this @@ -45,6 +46,6 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); void ScriptToUniv(const CScript& script, UniValue& out, bool include_address); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0); +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index 121e62457c..b5fc93886d 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -15,16 +15,15 @@ #include <version.h> #include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <algorithm> #include <string> -CScript ParseScript(const std::string& s) -{ - CScript result; +namespace { +opcodetype ParseOpCode(const std::string& s) +{ static std::map<std::string, opcodetype> mapOpNames; if (mapOpNames.empty()) @@ -40,11 +39,23 @@ CScript ParseScript(const std::string& s) continue; mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: - boost::algorithm::replace_first(strName, "OP_", ""); - mapOpNames[strName] = static_cast<opcodetype>(op); + if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" + mapOpNames[strName.substr(3)] = static_cast<opcodetype>(op); + } } } + auto it = mapOpNames.find(s); + if (it == mapOpNames.end()) throw std::runtime_error("script parse error: unknown opcode"); + return it->second; +} + +} // namespace + +CScript ParseScript(const std::string& s) +{ + CScript result; + std::vector<std::string> words; boost::algorithm::split(words, s, boost::algorithm::is_any_of(" \t\n"), boost::algorithm::token_compress_on); @@ -82,14 +93,10 @@ CScript ParseScript(const std::string& s) std::vector<unsigned char> value(w->begin()+1, w->end()-1); result << value; } - else if (mapOpNames.count(*w)) - { - // opcode, e.g. OP_ADD or ADD: - result << mapOpNames[*w]; - } else { - throw std::runtime_error("script parse error"); + // opcode, e.g. OP_ADD or ADD: + result << ParseOpCode(*w); } } @@ -119,31 +126,72 @@ static bool CheckTxScriptsSanity(const CMutableTransaction& tx) static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& tx_data, bool try_no_witness, bool try_witness) { + // General strategy: + // - Decode both with extended serialization (which interprets the 0x0001 tag as a marker for + // the presence of witnesses) and with legacy serialization (which interprets the tag as a + // 0-input 1-output incomplete transaction). + // - Restricted by try_no_witness (which disables legacy if false) and try_witness (which + // disables extended if false). + // - Ignore serializations that do not fully consume the hex string. + // - If neither succeeds, fail. + // - If only one succeeds, return that one. + // - If both decode attempts succeed: + // - If only one passes the CheckTxScriptsSanity check, return that one. + // - If neither or both pass CheckTxScriptsSanity, return the extended one. + + CMutableTransaction tx_extended, tx_legacy; + bool ok_extended = false, ok_legacy = false; + + // Try decoding with extended serialization support, and remember if the result successfully + // consumes the entire input. if (try_witness) { CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION); try { - ssData >> tx; - // If transaction looks sane, we don't try other mode even if requested - if (ssData.empty() && (!try_no_witness || CheckTxScriptsSanity(tx))) { - return true; - } + ssData >> tx_extended; + if (ssData.empty()) ok_extended = true; } catch (const std::exception&) { // Fall through. } } + // Optimization: if extended decoding succeeded and the result passes CheckTxScriptsSanity, + // don't bother decoding the other way. + if (ok_extended && CheckTxScriptsSanity(tx_extended)) { + tx = std::move(tx_extended); + return true; + } + + // Try decoding with legacy serialization, and remember if the result successfully consumes the entire input. if (try_no_witness) { CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); try { - ssData >> tx; - if (ssData.empty()) { - return true; - } + ssData >> tx_legacy; + if (ssData.empty()) ok_legacy = true; } catch (const std::exception&) { // Fall through. } } + // If legacy decoding succeeded and passes CheckTxScriptsSanity, that's our answer, as we know + // at this point that extended decoding either failed or doesn't pass the sanity check. + if (ok_legacy && CheckTxScriptsSanity(tx_legacy)) { + tx = std::move(tx_legacy); + return true; + } + + // If extended decoding succeeded, and neither decoding passes sanity, return the extended one. + if (ok_extended) { + tx = std::move(tx_extended); + return true; + } + + // If legacy decoding succeeded and extended didn't, return the legacy one. + if (ok_legacy) { + tx = std::move(tx_legacy); + return true; + } + + // If none succeeded, we failed. return false; } diff --git a/src/core_write.cpp b/src/core_write.cpp index 3980d8cb2e..a3902863d6 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,9 @@ #include <script/standard.h> #include <serialize.h> #include <streams.h> +#include <undo.h> #include <univalue.h> +#include <util/check.h> #include <util/system.h> #include <util/strencodings.h> @@ -177,7 +179,7 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey, out.pushKV("addresses", a); } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags) +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -189,13 +191,20 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, entry.pushKV("weight", GetTransactionWeight(tx)); entry.pushKV("locktime", (int64_t)tx.nLockTime); - UniValue vin(UniValue::VARR); + UniValue vin{UniValue::VARR}; + + // If available, use Undo data to calculate the fee. Note that txundo == nullptr + // for coinbase transactions and for transactions where undo data is unavailable. + const bool calculate_fee = txundo != nullptr; + CAmount amt_total_in = 0; + CAmount amt_total_out = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) { const CTxIn& txin = tx.vin[i]; UniValue in(UniValue::VOBJ); - if (tx.IsCoinBase()) + if (tx.IsCoinBase()) { in.pushKV("coinbase", HexStr(txin.scriptSig)); - else { + } else { in.pushKV("txid", txin.prevout.hash.GetHex()); in.pushKV("vout", (int64_t)txin.prevout.n); UniValue o(UniValue::VOBJ); @@ -210,6 +219,10 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, } in.pushKV("txinwitness", txinwitness); } + if (calculate_fee) { + const CTxOut& prev_txout = txundo->vprevout[i].out; + amt_total_in += prev_txout.nValue; + } in.pushKV("sequence", (int64_t)txin.nSequence); vin.push_back(in); } @@ -228,9 +241,19 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, ScriptPubKeyToUniv(txout.scriptPubKey, o, true); out.pushKV("scriptPubKey", o); vout.push_back(out); + + if (calculate_fee) { + amt_total_out += txout.nValue; + } } entry.pushKV("vout", vout); + if (calculate_fee) { + const CAmount fee = amt_total_in - amt_total_out; + CHECK_NONFATAL(MoneyRange(fee)); + entry.pushKV("fee", ValueFromAmount(fee)); + } + if (!hashBlock.IsNull()) entry.pushKV("blockhash", hashBlock.GetHex()); diff --git a/src/crc32c/.appveyor.yml b/src/crc32c/.appveyor.yml index 7345746750..b23e02e88a 100644 --- a/src/crc32c/.appveyor.yml +++ b/src/crc32c/.appveyor.yml @@ -8,9 +8,9 @@ environment: matrix: # AppVeyor currently has no custom job name feature. # http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs - - JOB: Visual Studio 2017 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - CMAKE_GENERATOR: Visual Studio 15 2017 + - JOB: Visual Studio 2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 platform: - x86 @@ -24,10 +24,11 @@ build_script: - git submodule update --init --recursive - mkdir build - cd build - - if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64 + - if "%platform%"=="x86" (set CMAKE_GENERATOR_PLATFORM="Win32") + else (set CMAKE_GENERATOR_PLATFORM="%platform%") - cmake --version - - cmake .. -G "%CMAKE_GENERATOR%" -DCRC32C_USE_GLOG=0 - -DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%" + - cmake .. -G "%CMAKE_GENERATOR%" -A "%CMAKE_GENERATOR_PLATFORM%" + -DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%" -DCRC32C_USE_GLOG=0 - cmake --build . --config "%CONFIGURATION%" - cd .. diff --git a/src/crc32c/AUTHORS b/src/crc32c/AUTHORS index 6f1f6871a6..ef9b4ea933 100644 --- a/src/crc32c/AUTHORS +++ b/src/crc32c/AUTHORS @@ -7,3 +7,5 @@ Google Inc. Fangming Fang <Fangming.Fang@arm.com> Vadim Skipin <vadim.skipin@gmail.com> +Rodrigo Tobar <rtobar@icrar.org> +Harry Mallon <hjmallon@gmail.com> diff --git a/src/crc32c/CMakeLists.txt b/src/crc32c/CMakeLists.txt index 111a3e3614..71692d5796 100644 --- a/src/crc32c/CMakeLists.txt +++ b/src/crc32c/CMakeLists.txt @@ -5,15 +5,21 @@ cmake_minimum_required(VERSION 3.1) project(Crc32c VERSION 1.1.0 LANGUAGES C CXX) -# This project can use C11, but will gracefully decay down to C89. -set(CMAKE_C_STANDARD 11) -set(CMAKE_C_STANDARD_REQUIRED OFF) -set(CMAKE_C_EXTENSIONS OFF) - -# This project requires C++11. -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +# C standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_C_STANDARD) + # This project can use C11, but will gracefully decay down to C89. + set(CMAKE_C_STANDARD 11) + set(CMAKE_C_STANDARD_REQUIRED OFF) + set(CMAKE_C_EXTENSIONS OFF) +endif(NOT CMAKE_C_STANDARD) + +# C++ standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_CXX_STANDARD) + # This project requires C++11. + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif(NOT CMAKE_CXX_STANDARD) # https://github.com/izenecloud/cmake/blob/master/SetCompilerWarningAll.cmake if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") @@ -269,7 +275,7 @@ target_sources(crc32c PRIVATE "${PROJECT_BINARY_DIR}/include/crc32c/crc32c_config.h" "src/crc32c_arm64.h" - "src/crc32c_arm64_linux_check.h" + "src/crc32c_arm64_check.h" "src/crc32c_internal.h" "src/crc32c_portable.cc" "src/crc32c_prefetch.h" @@ -405,19 +411,24 @@ if(CRC32C_INSTALL) ) include(CMakePackageConfigHelpers) + configure_package_config_file( + "${PROJECT_NAME}Config.cmake.in" + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) write_basic_package_version_file( - "${PROJECT_BINARY_DIR}/Crc32cConfigVersion.cmake" - COMPATIBILITY SameMajorVersion + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + COMPATIBILITY SameMajorVersion ) install( EXPORT Crc32cTargets NAMESPACE Crc32c:: - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Crc32c" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" ) install( FILES - "Crc32cConfig.cmake" - "${PROJECT_BINARY_DIR}/Crc32cConfigVersion.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Crc32c" + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" ) endif(CRC32C_INSTALL) diff --git a/src/crc32c/Crc32cConfig.cmake b/src/crc32c/Crc32cConfig.cmake.in index 4d6057ec26..c6b8fc7913 100644 --- a/src/crc32c/Crc32cConfig.cmake +++ b/src/crc32c/Crc32cConfig.cmake.in @@ -2,4 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. See the AUTHORS file for names of contributors. +@PACKAGE_INIT@ + include("${CMAKE_CURRENT_LIST_DIR}/Crc32cTargets.cmake") + +check_required_components(Crc32c) diff --git a/src/crc32c/src/crc32c.cc b/src/crc32c/src/crc32c.cc index 4d3018af47..804133bc17 100644 --- a/src/crc32c/src/crc32c.cc +++ b/src/crc32c/src/crc32c.cc @@ -8,7 +8,7 @@ #include <cstdint> #include "./crc32c_arm64.h" -#include "./crc32c_arm64_linux_check.h" +#include "./crc32c_arm64_check.h" #include "./crc32c_internal.h" #include "./crc32c_sse42.h" #include "./crc32c_sse42_check.h" @@ -20,8 +20,8 @@ uint32_t Extend(uint32_t crc, const uint8_t* data, size_t count) { static bool can_use_sse42 = CanUseSse42(); if (can_use_sse42) return ExtendSse42(crc, data, count); #elif HAVE_ARM64_CRC32C - static bool can_use_arm_linux = CanUseArm64Linux(); - if (can_use_arm_linux) return ExtendArm64(crc, data, count); + static bool can_use_arm64_crc32 = CanUseArm64Crc32(); + if (can_use_arm64_crc32) return ExtendArm64(crc, data, count); #endif // HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) return ExtendPortable(crc, data, count); diff --git a/src/crc32c/src/crc32c_arm64.cc b/src/crc32c/src/crc32c_arm64.cc index b872245f95..1da04ed34a 100644 --- a/src/crc32c/src/crc32c_arm64.cc +++ b/src/crc32c/src/crc32c_arm64.cc @@ -64,7 +64,7 @@ namespace crc32c { -uint32_t ExtendArm64(uint32_t crc, const uint8_t *buf, size_t size) { +uint32_t ExtendArm64(uint32_t crc, const uint8_t *data, size_t size) { int64_t length = size; uint32_t crc0, crc1, crc2, crc3; uint64_t t0, t1, t2; @@ -74,7 +74,6 @@ uint32_t ExtendArm64(uint32_t crc, const uint8_t *buf, size_t size) { const poly64_t k0 = 0x8d96551c, k1 = 0xbd6f81f8, k2 = 0xdcb17aa4; crc = crc ^ kCRC32Xor; - const uint8_t *p = reinterpret_cast<const uint8_t *>(buf); while (length >= KBYTES) { crc0 = crc; @@ -83,14 +82,14 @@ uint32_t ExtendArm64(uint32_t crc, const uint8_t *buf, size_t size) { crc3 = 0; // Process 1024 bytes in parallel. - CRC32C1024BYTES(p); + CRC32C1024BYTES(data); // Merge the 4 partial CRC32C values. t2 = (uint64_t)vmull_p64(crc2, k2); t1 = (uint64_t)vmull_p64(crc1, k1); t0 = (uint64_t)vmull_p64(crc0, k0); - crc = __crc32cd(crc3, *(uint64_t *)p); - p += sizeof(uint64_t); + crc = __crc32cd(crc3, *(uint64_t *)data); + data += sizeof(uint64_t); crc ^= __crc32cd(0, t2); crc ^= __crc32cd(0, t1); crc ^= __crc32cd(0, t0); @@ -99,23 +98,23 @@ uint32_t ExtendArm64(uint32_t crc, const uint8_t *buf, size_t size) { } while (length >= 8) { - crc = __crc32cd(crc, *(uint64_t *)p); - p += 8; + crc = __crc32cd(crc, *(uint64_t *)data); + data += 8; length -= 8; } if (length & 4) { - crc = __crc32cw(crc, *(uint32_t *)p); - p += 4; + crc = __crc32cw(crc, *(uint32_t *)data); + data += 4; } if (length & 2) { - crc = __crc32ch(crc, *(uint16_t *)p); - p += 2; + crc = __crc32ch(crc, *(uint16_t *)data); + data += 2; } if (length & 1) { - crc = __crc32cb(crc, *p); + crc = __crc32cb(crc, *data); } return crc ^ kCRC32Xor; diff --git a/src/crc32c/src/crc32c_arm64.h b/src/crc32c/src/crc32c_arm64.h index 100cd56ec8..e093687ddc 100644 --- a/src/crc32c/src/crc32c_arm64.h +++ b/src/crc32c/src/crc32c_arm64.h @@ -2,10 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -// Linux-specific code checking the availability for ARM CRC32C instructions. +// ARM-specific code -#ifndef CRC32C_CRC32C_ARM_LINUX_H_ -#define CRC32C_CRC32C_ARM_LINUX_H_ +#ifndef CRC32C_CRC32C_ARM_H_ +#define CRC32C_CRC32C_ARM_H_ #include <cstddef> #include <cstdint> @@ -24,4 +24,4 @@ uint32_t ExtendArm64(uint32_t crc, const uint8_t* data, size_t count); #endif // HAVE_ARM64_CRC32C -#endif // CRC32C_CRC32C_ARM_LINUX_H_ +#endif // CRC32C_CRC32C_ARM_H_ diff --git a/src/crc32c/src/crc32c_arm64_linux_check.h b/src/crc32c/src/crc32c_arm64_check.h index 1a20a757bb..62a07aba09 100644 --- a/src/crc32c/src/crc32c_arm64_linux_check.h +++ b/src/crc32c/src/crc32c_arm64_check.h @@ -2,12 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -// ARM Linux-specific code checking for the availability of CRC32C instructions. +// ARM-specific code checking for the availability of CRC32C instructions. -#ifndef CRC32C_CRC32C_ARM_LINUX_CHECK_H_ -#define CRC32C_CRC32C_ARM_LINUX_CHECK_H_ - -// X86-specific code checking for the availability of SSE4.2 instructions. +#ifndef CRC32C_CRC32C_ARM_CHECK_H_ +#define CRC32C_CRC32C_ARM_CHECK_H_ #include <cstddef> #include <cstdint> @@ -18,6 +16,7 @@ #if HAVE_ARM64_CRC32C +#ifdef __linux__ #if HAVE_STRONG_GETAUXVAL #include <sys/auxv.h> #elif HAVE_WEAK_GETAUXVAL @@ -27,17 +26,28 @@ extern "C" unsigned long getauxval(unsigned long type) __attribute__((weak)); #define AT_HWCAP 16 #endif // HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL +#endif // defined (__linux__) + +#ifdef __APPLE__ +#include <sys/types.h> +#include <sys/sysctl.h> +#endif // defined (__APPLE__) namespace crc32c { -inline bool CanUseArm64Linux() { -#if HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL +inline bool CanUseArm64Crc32() { +#if defined (__linux__) && (HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL) // From 'arch/arm64/include/uapi/asm/hwcap.h' in Linux kernel source code. constexpr unsigned long kHWCAP_PMULL = 1 << 4; constexpr unsigned long kHWCAP_CRC32 = 1 << 7; unsigned long hwcap = (&getauxval != nullptr) ? getauxval(AT_HWCAP) : 0; return (hwcap & (kHWCAP_PMULL | kHWCAP_CRC32)) == (kHWCAP_PMULL | kHWCAP_CRC32); +#elif defined(__APPLE__) + int val = 0; + size_t len = sizeof(val); + return sysctlbyname("hw.optional.armv8_crc32", &val, &len, nullptr, 0) == 0 + && val != 0; #else return false; #endif // HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL @@ -47,4 +57,4 @@ inline bool CanUseArm64Linux() { #endif // HAVE_ARM64_CRC32C -#endif // CRC32C_CRC32C_ARM_LINUX_CHECK_H_ +#endif // CRC32C_CRC32C_ARM_CHECK_H_ diff --git a/src/crc32c/src/crc32c_benchmark.cc b/src/crc32c/src/crc32c_benchmark.cc index c464304b3f..51194b370a 100644 --- a/src/crc32c/src/crc32c_benchmark.cc +++ b/src/crc32c/src/crc32c_benchmark.cc @@ -16,7 +16,7 @@ #endif // CRC32C_TESTS_BUILT_WITH_GLOG #include "./crc32c_arm64.h" -#include "./crc32c_arm64_linux_check.h" +#include "./crc32c_arm64_check.h" #include "./crc32c_internal.h" #include "./crc32c_sse42.h" #include "./crc32c_sse42_check.h" @@ -58,8 +58,8 @@ BENCHMARK_REGISTER_F(CRC32CBenchmark, Portable) #if HAVE_ARM64_CRC32C -BENCHMARK_DEFINE_F(CRC32CBenchmark, ArmLinux)(benchmark::State& state) { - if (!crc32c::CanUseArm64Linux()) { +BENCHMARK_DEFINE_F(CRC32CBenchmark, ArmCRC32C)(benchmark::State& state) { + if (!crc32c::CanUseArm64Crc32()) { state.SkipWithError("ARM CRC32C instructions not available or not enabled"); return; } @@ -69,7 +69,7 @@ BENCHMARK_DEFINE_F(CRC32CBenchmark, ArmLinux)(benchmark::State& state) { crc = crc32c::ExtendArm64(crc, block_buffer_, block_size_); state.SetBytesProcessed(state.iterations() * block_size_); } -BENCHMARK_REGISTER_F(CRC32CBenchmark, ArmLinux) +BENCHMARK_REGISTER_F(CRC32CBenchmark, ArmCRC32C) ->RangeMultiplier(16) ->Range(256, 16777216); // Block size. diff --git a/src/crc32c/src/crc32c_read_le.h b/src/crc32c/src/crc32c_read_le.h index 3bd45fe3aa..673a2a0db7 100644 --- a/src/crc32c/src/crc32c_read_le.h +++ b/src/crc32c/src/crc32c_read_le.h @@ -32,14 +32,14 @@ inline uint32_t ReadUint32LE(const uint8_t* buffer) { // Reads a little-endian 64-bit integer from a 64-bit-aligned buffer. inline uint64_t ReadUint64LE(const uint8_t* buffer) { #if BYTE_ORDER_BIG_ENDIAN - return ((static_cast<uint32_t>(static_cast<uint8_t>(buffer[0]))) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[1])) << 8) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[2])) << 16) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[3])) << 24) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[4])) << 32) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[5])) << 40) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[6])) << 48) | - (static_cast<uint32_t>(static_cast<uint8_t>(buffer[7])) << 56)); + return ((static_cast<uint64_t>(static_cast<uint8_t>(buffer[0]))) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[1])) << 8) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[2])) << 16) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[3])) << 24) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[4])) << 32) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[5])) << 40) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[6])) << 48) | + (static_cast<uint64_t>(static_cast<uint8_t>(buffer[7])) << 56)); #else // !BYTE_ORDER_BIG_ENDIAN uint64_t result; // This should be optimized to a single instruction. diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h index b3ba781cdd..0afe8fcc14 100644 --- a/src/crypto/chacha_poly_aead.h +++ b/src/crypto/chacha_poly_aead.h @@ -17,12 +17,12 @@ static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ /* A AEAD class for ChaCha20-Poly1305@bitcoin. * * ChaCha20 is a stream cipher designed by Daniel Bernstein and described in - * <ref>[http://cr.yp.to/chacha/chacha-20080128.pdf ChaCha20]</ref>. It operates + * <ref>[https://cr.yp.to/chacha/chacha-20080128.pdf ChaCha20]</ref>. It operates * by permuting 128 fixed bits, 128 or 256 bits of key, a 64 bit nonce and a 64 * bit counter into 64 bytes of output. This output is used as a keystream, with * any unused bytes simply discarded. * - * Poly1305 <ref>[http://cr.yp.to/mac/poly1305-20050329.pdf Poly1305]</ref>, also + * Poly1305 <ref>[https://cr.yp.to/mac/poly1305-20050329.pdf Poly1305]</ref>, also * by Daniel Bernstein, is a one-time Carter-Wegman MAC that computes a 128 bit * integrity tag given a message and a single-use 256 bit secret key. * diff --git a/src/crypto/common.h b/src/crypto/common.h index c1acf8b22e..dc12ed9942 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp new file mode 100644 index 0000000000..fbd14f9325 --- /dev/null +++ b/src/crypto/muhash.cpp @@ -0,0 +1,338 @@ +// Copyright (c) 2017-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/muhash.h> + +#include <crypto/chacha20.h> +#include <crypto/common.h> +#include <hash.h> + +#include <cassert> +#include <cstdio> +#include <limits> + +namespace { + +using limb_t = Num3072::limb_t; +using double_limb_t = Num3072::double_limb_t; +constexpr int LIMB_SIZE = Num3072::LIMB_SIZE; +constexpr int LIMBS = Num3072::LIMBS; +/** 2^3072 - 1103717, the largest 3072-bit safe prime number, is used as the modulus. */ +constexpr limb_t MAX_PRIME_DIFF = 1103717; + +/** Extract the lowest limb of [c0,c1,c2] into n, and left shift the number by 1 limb. */ +inline void extract3(limb_t& c0, limb_t& c1, limb_t& c2, limb_t& n) +{ + n = c0; + c0 = c1; + c1 = c2; + c2 = 0; +} + +/** [c0,c1] = a * b */ +inline void mul(limb_t& c0, limb_t& c1, const limb_t& a, const limb_t& b) +{ + double_limb_t t = (double_limb_t)a * b; + c1 = t >> LIMB_SIZE; + c0 = t; +} + +/* [c0,c1,c2] += n * [d0,d1,d2]. c2 is 0 initially */ +inline void mulnadd3(limb_t& c0, limb_t& c1, limb_t& c2, limb_t& d0, limb_t& d1, limb_t& d2, const limb_t& n) +{ + double_limb_t t = (double_limb_t)d0 * n + c0; + c0 = t; + t >>= LIMB_SIZE; + t += (double_limb_t)d1 * n + c1; + c1 = t; + t >>= LIMB_SIZE; + c2 = t + d2 * n; +} + +/* [c0,c1] *= n */ +inline void muln2(limb_t& c0, limb_t& c1, const limb_t& n) +{ + double_limb_t t = (double_limb_t)c0 * n; + c0 = t; + t >>= LIMB_SIZE; + t += (double_limb_t)c1 * n; + c1 = t; +} + +/** [c0,c1,c2] += a * b */ +inline void muladd3(limb_t& c0, limb_t& c1, limb_t& c2, const limb_t& a, const limb_t& b) +{ + double_limb_t t = (double_limb_t)a * b; + limb_t th = t >> LIMB_SIZE; + limb_t tl = t; + + c0 += tl; + th += (c0 < tl) ? 1 : 0; + c1 += th; + c2 += (c1 < th) ? 1 : 0; +} + +/** [c0,c1,c2] += 2 * a * b */ +inline void muldbladd3(limb_t& c0, limb_t& c1, limb_t& c2, const limb_t& a, const limb_t& b) +{ + double_limb_t t = (double_limb_t)a * b; + limb_t th = t >> LIMB_SIZE; + limb_t tl = t; + + c0 += tl; + limb_t tt = th + ((c0 < tl) ? 1 : 0); + c1 += tt; + c2 += (c1 < tt) ? 1 : 0; + c0 += tl; + th += (c0 < tl) ? 1 : 0; + c1 += th; + c2 += (c1 < th) ? 1 : 0; +} + +/** + * Add limb a to [c0,c1]: [c0,c1] += a. Then extract the lowest + * limb of [c0,c1] into n, and left shift the number by 1 limb. + * */ +inline void addnextract2(limb_t& c0, limb_t& c1, const limb_t& a, limb_t& n) +{ + limb_t c2 = 0; + + // add + c0 += a; + if (c0 < a) { + c1 += 1; + + // Handle case when c1 has overflown + if (c1 == 0) + c2 = 1; + } + + // extract + n = c0; + c0 = c1; + c1 = c2; +} + +/** in_out = in_out^(2^sq) * mul */ +inline void square_n_mul(Num3072& in_out, const int sq, const Num3072& mul) +{ + for (int j = 0; j < sq; ++j) in_out.Square(); + in_out.Multiply(mul); +} + +} // namespace + +/** Indicates wether d is larger than the modulus. */ +bool Num3072::IsOverflow() const +{ + if (this->limbs[0] <= std::numeric_limits<limb_t>::max() - MAX_PRIME_DIFF) return false; + for (int i = 1; i < LIMBS; ++i) { + if (this->limbs[i] != std::numeric_limits<limb_t>::max()) return false; + } + return true; +} + +void Num3072::FullReduce() +{ + limb_t c0 = MAX_PRIME_DIFF; + limb_t c1 = 0; + for (int i = 0; i < LIMBS; ++i) { + addnextract2(c0, c1, this->limbs[i], this->limbs[i]); + } +} + +Num3072 Num3072::GetInverse() const +{ + // For fast exponentiation a sliding window exponentiation with repunit + // precomputation is utilized. See "Fast Point Decompression for Standard + // Elliptic Curves" (Brumley, Järvinen, 2008). + + Num3072 p[12]; // p[i] = a^(2^(2^i)-1) + Num3072 out; + + p[0] = *this; + + for (int i = 0; i < 11; ++i) { + p[i + 1] = p[i]; + for (int j = 0; j < (1 << i); ++j) p[i + 1].Square(); + p[i + 1].Multiply(p[i]); + } + + out = p[11]; + + square_n_mul(out, 512, p[9]); + square_n_mul(out, 256, p[8]); + square_n_mul(out, 128, p[7]); + square_n_mul(out, 64, p[6]); + square_n_mul(out, 32, p[5]); + square_n_mul(out, 8, p[3]); + square_n_mul(out, 2, p[1]); + square_n_mul(out, 1, p[0]); + square_n_mul(out, 5, p[2]); + square_n_mul(out, 3, p[0]); + square_n_mul(out, 2, p[0]); + square_n_mul(out, 4, p[0]); + square_n_mul(out, 4, p[1]); + square_n_mul(out, 3, p[0]); + + return out; +} + +void Num3072::Multiply(const Num3072& a) +{ + limb_t c0 = 0, c1 = 0, c2 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of this*a into tmp, including one reduction. */ + for (int j = 0; j < LIMBS - 1; ++j) { + limb_t d0 = 0, d1 = 0, d2 = 0; + mul(d0, d1, this->limbs[1 + j], a.limbs[LIMBS + j - (1 + j)]); + for (int i = 2 + j; i < LIMBS; ++i) muladd3(d0, d1, d2, this->limbs[i], a.limbs[LIMBS + j - i]); + mulnadd3(c0, c1, c2, d0, d1, d2, MAX_PRIME_DIFF); + for (int i = 0; i < j + 1; ++i) muladd3(c0, c1, c2, this->limbs[i], a.limbs[j - i]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + + /* Compute limb N-1 of a*b into tmp. */ + assert(c2 == 0); + for (int i = 0; i < LIMBS; ++i) muladd3(c0, c1, c2, this->limbs[i], a.limbs[LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[LIMBS - 1]); + + /* Perform a second reduction. */ + muln2(c0, c1, MAX_PRIME_DIFF); + for (int j = 0; j < LIMBS; ++j) { + addnextract2(c0, c1, tmp.limbs[j], this->limbs[j]); + } + + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + + /* Perform up to two more reductions if the internal state has already + * overflown the MAX of Num3072 or if it is larger than the modulus or + * if both are the case. + * */ + if (this->IsOverflow()) this->FullReduce(); + if (c0) this->FullReduce(); +} + +void Num3072::Square() +{ + limb_t c0 = 0, c1 = 0, c2 = 0; + Num3072 tmp; + + /* Compute limbs 0..N-2 of this*this into tmp, including one reduction. */ + for (int j = 0; j < LIMBS - 1; ++j) { + limb_t d0 = 0, d1 = 0, d2 = 0; + for (int i = 0; i < (LIMBS - 1 - j) / 2; ++i) muldbladd3(d0, d1, d2, this->limbs[i + j + 1], this->limbs[LIMBS - 1 - i]); + if ((j + 1) & 1) muladd3(d0, d1, d2, this->limbs[(LIMBS - 1 - j) / 2 + j + 1], this->limbs[LIMBS - 1 - (LIMBS - 1 - j) / 2]); + mulnadd3(c0, c1, c2, d0, d1, d2, MAX_PRIME_DIFF); + for (int i = 0; i < (j + 1) / 2; ++i) muldbladd3(c0, c1, c2, this->limbs[i], this->limbs[j - i]); + if ((j + 1) & 1) muladd3(c0, c1, c2, this->limbs[(j + 1) / 2], this->limbs[j - (j + 1) / 2]); + extract3(c0, c1, c2, tmp.limbs[j]); + } + + assert(c2 == 0); + for (int i = 0; i < LIMBS / 2; ++i) muldbladd3(c0, c1, c2, this->limbs[i], this->limbs[LIMBS - 1 - i]); + extract3(c0, c1, c2, tmp.limbs[LIMBS - 1]); + + /* Perform a second reduction. */ + muln2(c0, c1, MAX_PRIME_DIFF); + for (int j = 0; j < LIMBS; ++j) { + addnextract2(c0, c1, tmp.limbs[j], this->limbs[j]); + } + + assert(c1 == 0); + assert(c0 == 0 || c0 == 1); + + /* Perform up to two more reductions if the internal state has already + * overflown the MAX of Num3072 or if it is larger than the modulus or + * if both are the case. + * */ + if (this->IsOverflow()) this->FullReduce(); + if (c0) this->FullReduce(); +} + +void Num3072::SetToOne() +{ + this->limbs[0] = 1; + for (int i = 1; i < LIMBS; ++i) this->limbs[i] = 0; +} + +void Num3072::Divide(const Num3072& a) +{ + if (this->IsOverflow()) this->FullReduce(); + + Num3072 inv{}; + if (a.IsOverflow()) { + Num3072 b = a; + b.FullReduce(); + inv = b.GetInverse(); + } else { + inv = a.GetInverse(); + } + + this->Multiply(inv); + if (this->IsOverflow()) this->FullReduce(); +} + +Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { + Num3072 out{}; + uint256 hashed_in = (CHashWriter(SER_DISK, 0) << in).GetSHA256(); + unsigned char tmp[BYTE_SIZE]; + ChaCha20(hashed_in.data(), hashed_in.size()).Keystream(tmp, BYTE_SIZE); + for (int i = 0; i < LIMBS; ++i) { + if (sizeof(limb_t) == 4) { + out.limbs[i] = ReadLE32(tmp + 4 * i); + } else if (sizeof(limb_t) == 8) { + out.limbs[i] = ReadLE64(tmp + 8 * i); + } + } + return out; +} + +MuHash3072::MuHash3072(Span<const unsigned char> in) noexcept +{ + m_numerator = ToNum3072(in); +} + +void MuHash3072::Finalize(uint256& out) noexcept +{ + m_numerator.Divide(m_denominator); + m_denominator.SetToOne(); // Needed to keep the MuHash object valid + + unsigned char data[384]; + for (int i = 0; i < LIMBS; ++i) { + if (sizeof(limb_t) == 4) { + WriteLE32(data + i * 4, m_numerator.limbs[i]); + } else if (sizeof(limb_t) == 8) { + WriteLE64(data + i * 8, m_numerator.limbs[i]); + } + } + + out = (CHashWriter(SER_DISK, 0) << data).GetSHA256(); +} + +MuHash3072& MuHash3072::operator*=(const MuHash3072& mul) noexcept +{ + m_numerator.Multiply(mul.m_numerator); + m_denominator.Multiply(mul.m_denominator); + return *this; +} + +MuHash3072& MuHash3072::operator/=(const MuHash3072& div) noexcept +{ + m_numerator.Multiply(div.m_denominator); + m_denominator.Multiply(div.m_numerator); + return *this; +} + +MuHash3072& MuHash3072::Insert(Span<const unsigned char> in) noexcept { + m_numerator.Multiply(ToNum3072(in)); + return *this; +} + +MuHash3072& MuHash3072::Remove(Span<const unsigned char> in) noexcept { + m_numerator.Divide(ToNum3072(in)); + return *this; +} diff --git a/src/crypto/muhash.h b/src/crypto/muhash.h new file mode 100644 index 0000000000..0c710007c4 --- /dev/null +++ b/src/crypto/muhash.h @@ -0,0 +1,130 @@ +// Copyright (c) 2017-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_MUHASH_H +#define BITCOIN_CRYPTO_MUHASH_H + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <serialize.h> +#include <uint256.h> + +#include <stdint.h> + +class Num3072 +{ +private: + void FullReduce(); + bool IsOverflow() const; + Num3072 GetInverse() const; + +public: + +#ifdef HAVE___INT128 + typedef unsigned __int128 double_limb_t; + typedef uint64_t limb_t; + static constexpr int LIMBS = 48; + static constexpr int LIMB_SIZE = 64; +#else + typedef uint64_t double_limb_t; + typedef uint32_t limb_t; + static constexpr int LIMBS = 96; + static constexpr int LIMB_SIZE = 32; +#endif + limb_t limbs[LIMBS]; + + // Sanity check for Num3072 constants + static_assert(LIMB_SIZE * LIMBS == 3072, "Num3072 isn't 3072 bits"); + static_assert(sizeof(double_limb_t) == sizeof(limb_t) * 2, "bad size for double_limb_t"); + static_assert(sizeof(limb_t) * 8 == LIMB_SIZE, "LIMB_SIZE is incorrect"); + + // Hard coded values in MuHash3072 constructor and Finalize + static_assert(sizeof(limb_t) == 4 || sizeof(limb_t) == 8, "bad size for limb_t"); + + void Multiply(const Num3072& a); + void Divide(const Num3072& a); + void SetToOne(); + void Square(); + + Num3072() { this->SetToOne(); }; + + SERIALIZE_METHODS(Num3072, obj) + { + for (auto& limb : obj.limbs) { + READWRITE(limb); + } + } +}; + +/** A class representing MuHash sets + * + * MuHash is a hashing algorithm that supports adding set elements in any + * order but also deleting in any order. As a result, it can maintain a + * running sum for a set of data as a whole, and add/remove when data + * is added to or removed from it. A downside of MuHash is that computing + * an inverse is relatively expensive. This is solved by representing + * the running value as a fraction, and multiplying added elements into + * the numerator and removed elements into the denominator. Only when the + * final hash is desired, a single modular inverse and multiplication is + * needed to combine the two. The combination is also run on serialization + * to allow for space-efficient storage on disk. + * + * As the update operations are also associative, H(a)+H(b)+H(c)+H(d) can + * in fact be computed as (H(a)+H(b)) + (H(c)+H(d)). This implies that + * all of this is perfectly parallellizable: each thread can process an + * arbitrary subset of the update operations, allowing them to be + * efficiently combined later. + * + * Muhash does not support checking if an element is already part of the + * set. That is why this class does not enforce the use of a set as the + * data it represents because there is no efficient way to do so. + * It is possible to add elements more than once and also to remove + * elements that have not been added before. However, this implementation + * is intended to represent a set of elements. + * + * See also https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and + * https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html. + */ +class MuHash3072 +{ +private: + static constexpr size_t BYTE_SIZE = 384; + + Num3072 m_numerator; + Num3072 m_denominator; + + Num3072 ToNum3072(Span<const unsigned char> in); + +public: + /* The empty set. */ + MuHash3072() noexcept {}; + + /* A singleton with variable sized data in it. */ + explicit MuHash3072(Span<const unsigned char> in) noexcept; + + /* Insert a single piece of data into the set. */ + MuHash3072& Insert(Span<const unsigned char> in) noexcept; + + /* Remove a single piece of data from the set. */ + MuHash3072& Remove(Span<const unsigned char> in) noexcept; + + /* Multiply (resulting in a hash for the union of the sets) */ + MuHash3072& operator*=(const MuHash3072& mul) noexcept; + + /* Divide (resulting in a hash for the difference of the sets) */ + MuHash3072& operator/=(const MuHash3072& div) noexcept; + + /* Finalize into a 32-byte hash. Does not change this object's value. */ + void Finalize(uint256& out) noexcept; + + SERIALIZE_METHODS(MuHash3072, obj) + { + READWRITE(obj.m_numerator); + READWRITE(obj.m_denominator); + } +}; + +#endif // BITCOIN_CRYPTO_MUHASH_H diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_shani.cpp index 3473f6e39f..4f4d5b5837 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_shani.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp index 89f529a3ab..143752c7cf 100644 --- a/src/crypto/sha256_sse4.cpp +++ b/src/crypto/sha256_sse4.cpp @@ -1001,7 +1001,7 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) ; This code is described in an Intel White-Paper: ; "Fast SHA-256 Implementations on Intel Architecture Processors" ; -; To find it, surf to http://www.intel.com/p/en_US/embedded +; To find it, surf to https://www.intel.com/p/en_US/embedded ; and search for that title. ; The paper is expected to be released roughly at the end of April, 2012 ; diff --git a/src/crypto/siphash.cpp b/src/crypto/siphash.cpp index 2e0106b165..2e90c393e1 100644 --- a/src/crypto/siphash.cpp +++ b/src/crypto/siphash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/siphash.h b/src/crypto/siphash.h index 6b38950f8e..b573526932 100644 --- a/src/crypto/siphash.h +++ b/src/crypto/siphash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 2daf676c4a..1166466771 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -225,7 +225,7 @@ private: * [0, 1) and simply multiply it by the size. Then we just shift the result down by * 32-bits to get our bucket number. The result has non-uniformity the same as a * mod, but it is much faster to compute. More about this technique can be found at - * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ . + * https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ . * * The resulting non-uniformity is also more equally distributed which would be * advantageous for something like linear probing, though it shouldn't matter diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 215b033708..33e4b366a1 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 8a8f7b681c..11cf357f3d 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -92,6 +92,7 @@ bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) fclose(file); return error("%s: failed to commit file %d", __func__, pos.nFile); } + DirectoryCommit(m_dir); fclose(file); return true; diff --git a/src/fs.cpp b/src/fs.cpp index eef9c81de9..7eb0f5ca9f 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/hash.cpp b/src/hash.cpp index 3657b38639..cc46043c2b 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ inline uint32_t ROTL32(uint32_t x, int8_t r) unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vDataToHash) { - // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + // The following is MurmurHash3 (x86_32), see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp uint32_t h1 = nHashSeed; const uint32_t c1 = 0xcc9e2d51; const uint32_t c2 = 0x1b873593; diff --git a/src/hash.h b/src/hash.h index 6d876076ee..1456a899d8 100644 --- a/src/hash.h +++ b/src/hash.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -197,7 +197,7 @@ uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL } /** Single-SHA256 a 32-byte input (represented as uint256). */ -NODISCARD uint256 SHA256Uint256(const uint256& input); +[[nodiscard]] uint256 SHA256Uint256(const uint256& input); unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vDataToHash); diff --git a/src/httprpc.cpp b/src/httprpc.cpp index f1b9997371..cb8b220895 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -68,6 +68,8 @@ private: static std::string strRPCUserColonPass; /* Stored RPC timer interface (for unregistration) */ static std::unique_ptr<HTTPRPCTimerInterface> httpRPCTimerInterface; +/* List of -rpcauth values */ +static std::vector<std::vector<std::string>> g_rpcauth; /* RPC Auth Whitelist */ static std::map<std::string, std::set<std::string>> g_rpc_whitelist; static bool g_rpc_whitelist_default = false; @@ -99,15 +101,7 @@ static bool multiUserAuthorized(std::string strUserPass) std::string strUser = strUserPass.substr(0, strUserPass.find(':')); std::string strPass = strUserPass.substr(strUserPass.find(':') + 1); - for (const std::string& strRPCAuth : gArgs.GetArgs("-rpcauth")) { - //Search for multi-user login/pass "rpcauth" from config - std::vector<std::string> vFields; - boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); - if (vFields.size() != 3) { - //Incorrect formatting in config file - continue; - } - + for (const auto& vFields : g_rpcauth) { std::string strName = vFields[0]; if (!TimingResistantEqual(strName, strUser)) { continue; @@ -259,6 +253,16 @@ static bool InitRPCAuthentication() if (gArgs.GetArg("-rpcauth","") != "") { LogPrintf("Using rpcauth authentication.\n"); + for (const std::string& rpcauth : gArgs.GetArgs("-rpcauth")) { + std::vector<std::string> fields; + boost::split(fields, rpcauth, boost::is_any_of(":$")); + if (fields.size() == 3) { + g_rpcauth.push_back(fields); + } else { + LogPrintf("Invalid -rpcauth argument.\n"); + return false; + } + } } g_rpc_whitelist_default = gArgs.GetBoolArg("-rpcwhitelistdefault", gArgs.IsArgSet("-rpcwhitelist")); diff --git a/src/httprpc.h b/src/httprpc.h index a6a38fc95a..97af6f7bb1 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 1e5ea2de83..0a8e58ab67 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -33,13 +33,6 @@ #include <support/events.h> -#ifdef EVENT__HAVE_NETINET_IN_H -#include <netinet/in.h> -#ifdef _XOPEN_SOURCE_EXTENDED -#include <arpa/inet.h> -#endif -#endif - /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 65a5f03a8e..4f61bbeabd 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,7 +51,6 @@ struct DBVal { struct DBHeightKey { int height; - DBHeightKey() : height(0) {} explicit DBHeightKey(int height_in) : height(height_in) {} template<typename Stream> diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 317f8c0e40..221ac02c9e 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,15 +9,11 @@ #include <chain.h> #include <flatfile.h> #include <index/base.h> +#include <util/hasher.h> /** Interval between compact filter checkpoints. See BIP 157. */ static constexpr int CFCHECKPT_INTERVAL = 1000; -struct FilterHeaderHasher -{ - size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } -}; - /** * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of * blocks by height. An index is constructed for each supported filter type with its own database diff --git a/src/index/disktxpos.h b/src/index/disktxpos.h index 69696b0ec5..3166053226 100644 --- a/src/index/disktxpos.h +++ b/src/index/disktxpos.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 462ac5962f..6398b7edc8 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index 495d96f938..09eb76eaee 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -26,6 +26,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> #include <key.h> +#include <mapport.h> #include <miner.h> #include <net.h> #include <net_permissions.h> @@ -85,7 +86,6 @@ #include <zmq/zmqrpc.h> #endif -static bool fFeeEstimatesInitialized = false; static const bool DEFAULT_PROXYRANDOMIZE = true; static const bool DEFAULT_REST_ENABLE = false; static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; @@ -99,8 +99,6 @@ static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; #define MIN_CORE_FILEDESCRIPTORS 150 #endif -static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; - static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; /** @@ -113,7 +111,7 @@ static fs::path GetPidFile(const ArgsManager& args) return AbsPathForConfigVal(fs::path(args.GetArg("-pid", BITCOIN_PID_FILENAME))); } -NODISCARD static bool CreatePidFile(const ArgsManager& args) +[[nodiscard]] static bool CreatePidFile(const ArgsManager& args) { fsbridge::ofstream file{GetPidFile(args)}; if (file) { @@ -203,7 +201,7 @@ void Shutdown(NodeContext& node) // using the other before destroying them. if (node.peerman) UnregisterValidationInterface(node.peerman.get()); // Follow the lock order requirements: - // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount + // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraFullOutboundCount // which locks cs_vNodes. // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which // locks cs_vNodes. @@ -236,17 +234,8 @@ void Shutdown(NodeContext& node) DumpMempool(*node.mempool); } - if (fFeeEstimatesInitialized) - { - ::feeEstimator.FlushUnconfirmed(); - fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); - if (!est_fileout.IsNull()) - ::feeEstimator.Write(est_fileout); - else - LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); - fFeeEstimatesInitialized = false; - } + // Drop transactions we were still watching, and record fee estimations. + if (node.fee_estimator) node.fee_estimator->Flush(); // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing if (node.chainman) { @@ -304,6 +293,7 @@ void Shutdown(NodeContext& node) globalVerifyHandle.reset(); ECC_Stop(); node.mempool.reset(); + node.fee_estimator.reset(); node.chainman = nullptr; node.scheduler.reset(); @@ -461,7 +451,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet: %u signet: %u, regtest: %u)", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections.", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -479,6 +469,11 @@ void SetupServerArgs(NodeContext& node) #else hidden_args.emplace_back("-upnp"); #endif +#ifdef USE_NATPMP + argsman.AddArg("-natpmp", strprintf("Use NAT-PMP to map the listening port (default: %s)", DEFAULT_NATPMP ? "1 when listening and no -proxy" : "0"), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); +#else + hidden_args.emplace_back("-natpmp"); +#endif // USE_NATPMP argsman.AddArg("-whitebind=<[permissions@]addr>", "Bind to the given address and add permission flags to the peers connecting to it. " "Use [host]:port notation for IPv6. Allowed permissions: " + Join(NET_PERMISSIONS_DOC, ", ") + ". " "Specify multiple permissions separated by commas (default: download,noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -519,7 +514,6 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-deprecatedrpc=<method>", "Allows deprecated RPC method(s) to be used", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-stopatheight", strprintf("Stop running after reaching the given height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -824,10 +818,13 @@ void InitParameterInteraction(ArgsManager& args) // to protect privacy, do not listen by default if a default proxy server is specified if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -listen=0\n", __func__); - // to protect privacy, do not use UPNP when a proxy is set. The user may still specify -listen=1 + // to protect privacy, do not map ports when a proxy is set. The user may still specify -listen=1 // to listen locally, so don't rely on this happening through -listen below. if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -upnp=0\n", __func__); + if (args.SoftSetBoolArg("-natpmp", false)) { + LogPrintf("%s: parameter interaction: -proxy set -> setting -natpmp=0\n", __func__); + } // to protect privacy, do not discover addresses by default if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -discover=0\n", __func__); @@ -837,6 +834,9 @@ void InitParameterInteraction(ArgsManager& args) // do not map ports or try to retrieve public IP when not listening (pointless) if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -upnp=0\n", __func__); + if (args.SoftSetBoolArg("-natpmp", false)) { + LogPrintf("%s: parameter interaction: -listen=0 -> setting -natpmp=0\n", __func__); + } if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -discover=0\n", __func__); if (args.SoftSetBoolArg("-listenonion", false)) @@ -914,7 +914,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup(ArgsManager& args) +bool AppInitBasicSetup(const ArgsManager& args) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -928,6 +928,9 @@ bool AppInitBasicSetup(ArgsManager& args) // Enable heap terminate-on-corruption HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif + if (!InitShutdownState()) { + return InitError(Untranslated("Initializing wait-for-shutdown state failed.")); + } if (!SetupNetworking()) { return InitError(Untranslated("Initializing networking failed.")); @@ -1039,7 +1042,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Trim requested connection counts, to fit into system limitations // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 - nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS); + nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind); #ifdef USE_POLL int fd_max = nFD; #else @@ -1384,27 +1387,31 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // is not yet setup and may end up being set up twice if we // need to reindex later. + fListen = args.GetBoolArg("-listen", DEFAULT_LISTEN); + fDiscover = args.GetBoolArg("-discover", true); + const bool ignores_incoming_txs{args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)}; + assert(!node.banman); node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, args.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), args.GetBoolArg("-networkactive", true)); - // Make mempool generally available in the node context. For example the connection manager, wallet, or RPC threads, - // which are all started after this, may use it from the node context. + assert(!node.fee_estimator); + // Don't initialize fee estimation with old data if we don't relay transactions, + // as they would never get updated. + if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); + assert(!node.mempool); - node.mempool = MakeUnique<CTxMemPool>(&::feeEstimator); - if (node.mempool) { - int ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); - if (ratio != 0) { - node.mempool->setSanityCheck(1.0 / ratio); - } - } + int check_ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); + node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), check_ratio); assert(!node.chainman); node.chainman = &g_chainman; ChainstateManager& chainman = *Assert(node.chainman); - node.peerman.reset(new PeerManager(chainparams, *node.connman, node.banman.get(), *node.scheduler, chainman, *node.mempool)); + assert(!node.peerman); + node.peerman = PeerManager::make(chainparams, *node.connman, node.banman.get(), + *node.scheduler, chainman, *node.mempool, ignores_incoming_txs); RegisterValidationInterface(node.peerman.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1480,11 +1487,6 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } } - // see Step 2: parameter interactions for more information about these - fListen = args.GetBoolArg("-listen", DEFAULT_LISTEN); - fDiscover = args.GetBoolArg("-discover", true); - g_relay_txes = !args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); - for (const std::string& strAddr : args.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) @@ -1525,12 +1527,6 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA RegisterValidationInterface(g_zmq_notification_interface); } #endif - uint64_t nMaxOutboundLimit = 0; //unlimited unless -maxuploadtarget is set - uint64_t nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; - - if (args.IsArgSet("-maxuploadtarget")) { - nMaxOutboundLimit = args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; - } // ********************************************************* Step 7: load block chain @@ -1792,13 +1788,6 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA return false; } - fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; - CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); - // Allowed to fail as this file IS missing on first startup. - if (!est_filein.IsNull()) - ::feeEstimator.Read(est_filein); - fFeeEstimatesInitialized = true; - // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex); @@ -1917,13 +1906,12 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA } } LogPrintf("nBestHeight = %d\n", chain_active_height); + if (node.peerman) node.peerman->SetBestHeight(chain_active_height); Discover(); - // Map ports with UPnP - if (args.GetBoolArg("-upnp", DEFAULT_UPNP)) { - StartMapPort(); - } + // Map ports with UPnP or NAT-PMP. + StartMapPort(args.GetBoolArg("-upnp", DEFAULT_UPNP), gArgs.GetBoolArg("-natpmp", DEFAULT_NATPMP)); CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; @@ -1932,7 +1920,6 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; - connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); @@ -1940,8 +1927,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA connOptions.nReceiveFloodSize = 1000 * args.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = args.GetArgs("-addnode"); - connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; - connOptions.nMaxOutboundLimit = nMaxOutboundLimit; + connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); connOptions.m_peer_connect_timeout = peer_connect_timeout; for (const std::string& bind_arg : args.GetArgs("-bind")) { diff --git a/src/init.h b/src/init.h index 679e875da1..c04d966d06 100644 --- a/src/init.h +++ b/src/init.h @@ -33,7 +33,7 @@ void InitParameterInteraction(ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(ArgsManager& args); +bool AppInitBasicSetup(const ArgsManager& args); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 85d09be0f3..1a49518d69 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -44,6 +44,10 @@ public: FoundBlock& time(int64_t& time) { m_time = &time; return *this; } FoundBlock& maxTime(int64_t& max_time) { m_max_time = &max_time; return *this; } FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } + //! Return whether block is in the active (most-work) chain. + FoundBlock& inActiveChain(bool& in_active_chain) { m_in_active_chain = &in_active_chain; return *this; } + //! Return next block in the active chain if current block is in the active chain. + FoundBlock& nextBlock(const FoundBlock& next_block) { m_next_block = &next_block; return *this; } //! Read block data from disk. If the block exists but doesn't have data //! (for example due to pruning), the CBlock variable will be set to null. FoundBlock& data(CBlock& data) { m_data = &data; return *this; } @@ -53,6 +57,8 @@ public: int64_t* m_time = nullptr; int64_t* m_max_time = nullptr; int64_t* m_mtp_time = nullptr; + bool* m_in_active_chain = nullptr; + const FoundBlock* m_next_block = nullptr; CBlock* m_data = nullptr; }; @@ -77,9 +83,9 @@ public: //! wallet cache it, fee estimation being driven by node mempool, wallet //! should be the consumer. //! -//! * The `guessVerificationProgress`, `getBlockHeight`, `getBlockHash`, etc -//! methods can go away if rescan logic is moved on the node side, and wallet -//! only register rescan request. +//! * `guessVerificationProgress` and similar methods can go away if rescan +//! logic moves out of the wallet, and the wallet just requests scans from the +//! node (https://github.com/bitcoin/bitcoin/issues/11756) class Chain { public: @@ -90,11 +96,6 @@ public: //! any blocks) virtual Optional<int> getHeight() = 0; - //! Get block height above genesis block. Returns 0 for genesis block, - //! 1 for following block, and so on. Returns nullopt for a block not - //! included in the current chain. - virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; @@ -102,13 +103,6 @@ public: //! pruned), and contains transactions. virtual bool haveBlockOnDisk(int height) = 0; - //! Return height of the first block in the chain with timestamp equal - //! or greater than the given time and height equal or greater than the - //! given height, or nullopt if there is no block with a high enough - //! timestamp and height. Also return the block hash as an optional output parameter - //! (to avoid the cost of a second lookup in case this information is needed.) - virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; @@ -130,11 +124,6 @@ public: //! information. virtual bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block={}) = 0; - //! Find next block if block is part of current chain. Also flag if - //! there was a reorg and the specified block hash is no longer in the - //! current chain, and optionally return block information. - virtual bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next={}, bool* reorg=nullptr) = 0; - //! Find ancestor of block at specified height and optionally return //! ancestor information. virtual bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out={}) = 0; diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp deleted file mode 100644 index 2c5f8627e6..0000000000 --- a/src/interfaces/node.cpp +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <interfaces/node.h> - -#include <addrdb.h> -#include <banman.h> -#include <chain.h> -#include <chainparams.h> -#include <init.h> -#include <interfaces/chain.h> -#include <interfaces/handler.h> -#include <interfaces/wallet.h> -#include <net.h> -#include <net_processing.h> -#include <netaddress.h> -#include <netbase.h> -#include <node/context.h> -#include <node/ui_interface.h> -#include <policy/feerate.h> -#include <policy/fees.h> -#include <policy/settings.h> -#include <primitives/block.h> -#include <rpc/server.h> -#include <shutdown.h> -#include <support/allocators/secure.h> -#include <sync.h> -#include <txmempool.h> -#include <util/check.h> -#include <util/ref.h> -#include <util/system.h> -#include <util/translation.h> -#include <validation.h> -#include <warnings.h> - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <univalue.h> - -#include <boost/signals2/signal.hpp> - -namespace interfaces { -namespace { - -class NodeImpl : public Node -{ -public: - NodeImpl(NodeContext* context) { setContext(context); } - void initLogging() override { InitLogging(*Assert(m_context->args)); } - void initParameterInteraction() override { InitParameterInteraction(*Assert(m_context->args)); } - bilingual_str getWarnings() override { return GetWarnings(true); } - uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } - bool baseInitialize() override - { - return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && - AppInitLockDataDirectory() && AppInitInterfaces(*m_context); - } - bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override - { - return AppInitMain(m_context_ref, *m_context, tip_info); - } - void appShutdown() override - { - Interrupt(*m_context); - Shutdown(*m_context); - } - void startShutdown() override - { - StartShutdown(); - // Stop RPC for clean shutdown if any of waitfor* commands is executed. - if (gArgs.GetBoolArg("-server", false)) { - InterruptRPC(); - StopRPC(); - } - } - bool shutdownRequested() override { return ShutdownRequested(); } - void mapPort(bool use_upnp) override - { - if (use_upnp) { - StartMapPort(); - } else { - InterruptMapPort(); - StopMapPort(); - } - } - bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } - size_t getNodeCount(CConnman::NumConnections flags) override - { - return m_context->connman ? m_context->connman->GetNodeCount(flags) : 0; - } - bool getNodesStats(NodesStats& stats) override - { - stats.clear(); - - if (m_context->connman) { - std::vector<CNodeStats> stats_temp; - m_context->connman->GetNodeStats(stats_temp); - - stats.reserve(stats_temp.size()); - for (auto& node_stats_temp : stats_temp) { - stats.emplace_back(std::move(node_stats_temp), false, CNodeStateStats()); - } - - // Try to retrieve the CNodeStateStats for each node. - TRY_LOCK(::cs_main, lockMain); - if (lockMain) { - for (auto& node_stats : stats) { - std::get<1>(node_stats) = - GetNodeStateStats(std::get<0>(node_stats).nodeid, std::get<2>(node_stats)); - } - } - return true; - } - return false; - } - bool getBanned(banmap_t& banmap) override - { - if (m_context->banman) { - m_context->banman->GetBanned(banmap); - return true; - } - return false; - } - bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override - { - if (m_context->banman) { - m_context->banman->Ban(net_addr, ban_time_offset); - return true; - } - return false; - } - bool unban(const CSubNet& ip) override - { - if (m_context->banman) { - m_context->banman->Unban(ip); - return true; - } - return false; - } - bool disconnectByAddress(const CNetAddr& net_addr) override - { - if (m_context->connman) { - return m_context->connman->DisconnectNode(net_addr); - } - return false; - } - bool disconnectById(NodeId id) override - { - if (m_context->connman) { - return m_context->connman->DisconnectNode(id); - } - return false; - } - int64_t getTotalBytesRecv() override { return m_context->connman ? m_context->connman->GetTotalBytesRecv() : 0; } - int64_t getTotalBytesSent() override { return m_context->connman ? m_context->connman->GetTotalBytesSent() : 0; } - size_t getMempoolSize() override { return m_context->mempool ? m_context->mempool->size() : 0; } - size_t getMempoolDynamicUsage() override { return m_context->mempool ? m_context->mempool->DynamicMemoryUsage() : 0; } - bool getHeaderTip(int& height, int64_t& block_time) override - { - LOCK(::cs_main); - if (::pindexBestHeader) { - height = ::pindexBestHeader->nHeight; - block_time = ::pindexBestHeader->GetBlockTime(); - return true; - } - return false; - } - int getNumBlocks() override - { - LOCK(::cs_main); - return ::ChainActive().Height(); - } - uint256 getBestBlockHash() override - { - const CBlockIndex* tip = WITH_LOCK(::cs_main, return ::ChainActive().Tip()); - return tip ? tip->GetBlockHash() : Params().GenesisBlock().GetHash(); - } - int64_t getLastBlockTime() override - { - LOCK(::cs_main); - if (::ChainActive().Tip()) { - return ::ChainActive().Tip()->GetBlockTime(); - } - return Params().GenesisBlock().GetBlockTime(); // Genesis block's time of current network - } - double getVerificationProgress() override - { - const CBlockIndex* tip; - { - LOCK(::cs_main); - tip = ::ChainActive().Tip(); - } - return GuessVerificationProgress(Params().TxData(), tip); - } - bool isInitialBlockDownload() override { return ::ChainstateActive().IsInitialBlockDownload(); } - bool getReindex() override { return ::fReindex; } - bool getImporting() override { return ::fImporting; } - void setNetworkActive(bool active) override - { - if (m_context->connman) { - m_context->connman->SetNetworkActive(active); - } - } - bool getNetworkActive() override { return m_context->connman && m_context->connman->GetNetworkActive(); } - CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override - { - FeeCalculation fee_calc; - CFeeRate result = ::feeEstimator.estimateSmartFee(num_blocks, &fee_calc, conservative); - if (returned_target) { - *returned_target = fee_calc.returnedTarget; - } - return result; - } - CFeeRate getDustRelayFee() override { return ::dustRelayFee; } - UniValue executeRpc(const std::string& command, const UniValue& params, const std::string& uri) override - { - JSONRPCRequest req(m_context_ref); - req.params = params; - req.strMethod = command; - req.URI = uri; - return ::tableRPC.execute(req); - } - std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); } - void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } - void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } - bool getUnspentOutput(const COutPoint& output, Coin& coin) override - { - LOCK(::cs_main); - return ::ChainstateActive().CoinsTip().GetCoin(output, coin); - } - WalletClient& walletClient() override - { - return *Assert(m_context->wallet_client); - } - std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) override - { - return MakeHandler(::uiInterface.InitMessage_connect(fn)); - } - std::unique_ptr<Handler> handleMessageBox(MessageBoxFn fn) override - { - return MakeHandler(::uiInterface.ThreadSafeMessageBox_connect(fn)); - } - std::unique_ptr<Handler> handleQuestion(QuestionFn fn) override - { - return MakeHandler(::uiInterface.ThreadSafeQuestion_connect(fn)); - } - std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override - { - return MakeHandler(::uiInterface.ShowProgress_connect(fn)); - } - std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override - { - return MakeHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); - } - std::unique_ptr<Handler> handleNotifyNetworkActiveChanged(NotifyNetworkActiveChangedFn fn) override - { - return MakeHandler(::uiInterface.NotifyNetworkActiveChanged_connect(fn)); - } - std::unique_ptr<Handler> handleNotifyAlertChanged(NotifyAlertChangedFn fn) override - { - return MakeHandler(::uiInterface.NotifyAlertChanged_connect(fn)); - } - std::unique_ptr<Handler> handleBannedListChanged(BannedListChangedFn fn) override - { - return MakeHandler(::uiInterface.BannedListChanged_connect(fn)); - } - std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override - { - return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { - fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, - GuessVerificationProgress(Params().TxData(), block)); - })); - } - std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override - { - return MakeHandler( - ::uiInterface.NotifyHeaderTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { - fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, - /* verification progress is unused when a header was received */ 0); - })); - } - NodeContext* context() override { return m_context; } - void setContext(NodeContext* context) override - { - m_context = context; - if (context) { - m_context_ref.Set(*context); - } else { - m_context_ref.Clear(); - } - } - NodeContext* m_context{nullptr}; - util::Ref m_context_ref; -}; - -} // namespace - -std::unique_ptr<Node> MakeNode(NodeContext* context) { return MakeUnique<NodeImpl>(context); } - -} // namespace interfaces diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 5079be038e..15f7ef6256 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -82,7 +82,7 @@ public: virtual bool shutdownRequested() = 0; //! Map port. - virtual void mapPort(bool use_upnp) = 0; + virtual void mapPort(bool use_upnp, bool use_natpmp) = 0; //! Get proxy. virtual bool getProxy(Network net, proxyType& proxy_info) = 0; @@ -151,9 +151,6 @@ public: //! Get network active. virtual bool getNetworkActive() = 0; - //! Estimate smart fee. - virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) = 0; - //! Get dust relay fee. virtual CFeeRate getDustRelayFee() = 0; diff --git a/src/key.cpp b/src/key.cpp index 868a8b9b0e..1e59b301cb 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ static secp256k1_context* secp256k1_context_sign = nullptr; /** * This parses a format loosely based on a DER encoding of the ECPrivateKey type from - * section C.4 of SEC 1 <http://www.secg.org/sec1-v2.pdf>, with the following caveats: + * section C.4 of SEC 1 <https://www.secg.org/sec1-v2.pdf>, with the following caveats: * * * The octet-length of the SEQUENCE must be encoded as 1 or 2 octets. It is not * required to be encoded as one octet if it is less than 256, as DER would require. @@ -80,7 +80,7 @@ int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char *out32, con /** * This serializes to a DER encoding of the ECPrivateKey type from section C.4 of SEC 1 - * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are + * <https://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are * included. * * seckey must point to an output buffer of length at least CKey::SIZE bytes. diff --git a/src/key_io.cpp b/src/key_io.cpp index d2f5be93f5..a270ede864 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -8,16 +8,12 @@ #include <bech32.h> #include <util/strencodings.h> -#include <boost/variant/apply_visitor.hpp> -#include <boost/variant/static_visitor.hpp> - +#include <algorithm> #include <assert.h> #include <string.h> -#include <algorithm> -namespace -{ -class DestinationEncoder : public boost::static_visitor<std::string> +namespace { +class DestinationEncoder { private: const CChainParams& m_params; @@ -209,7 +205,7 @@ std::string EncodeExtKey(const CExtKey& key) std::string EncodeDestination(const CTxDestination& dest) { - return boost::apply_visitor(DestinationEncoder(Params()), dest); + return std::visit(DestinationEncoder(Params()), dest); } CTxDestination DecodeDestination(const std::string& str) diff --git a/src/logging.cpp b/src/logging.cpp index 35e0754f20..4ddcf1d930 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -174,7 +174,7 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) return false; } -std::vector<LogCategory> BCLog::Logger::LogCategoriesList() +std::vector<LogCategory> BCLog::Logger::LogCategoriesList() const { std::vector<LogCategory> ret; for (const CLogCategoryDesc& category_desc : LogCategories) { diff --git a/src/logging.h b/src/logging.h index 7e646ef67a..9efecc7c12 100644 --- a/src/logging.h +++ b/src/logging.h @@ -135,9 +135,9 @@ namespace BCLog { bool WillLogCategory(LogFlags category) const; /** Returns a vector of the log categories */ - std::vector<LogCategory> LogCategoriesList(); + std::vector<LogCategory> LogCategoriesList() const; /** Returns a string with the log categories */ - std::string LogCategoriesString() + std::string LogCategoriesString() const { return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); }; diff --git a/src/mapport.cpp b/src/mapport.cpp new file mode 100644 index 0000000000..2df4ce45d2 --- /dev/null +++ b/src/mapport.cpp @@ -0,0 +1,336 @@ +// Copyright (c) 2011-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <mapport.h> + +#include <clientversion.h> +#include <logging.h> +#include <net.h> +#include <netaddress.h> +#include <netbase.h> +#include <threadinterrupt.h> +#include <util/system.h> + +#ifdef USE_NATPMP +#include <compat.h> +#include <natpmp.h> +#endif // USE_NATPMP + +#ifdef USE_UPNP +#include <miniupnpc/miniupnpc.h> +#include <miniupnpc/upnpcommands.h> +#include <miniupnpc/upnperrors.h> +// The minimum supported miniUPnPc API version is set to 10. This keeps compatibility +// with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. +static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed"); +#endif // USE_UPNP + +#include <atomic> +#include <cassert> +#include <chrono> +#include <functional> +#include <string> +#include <thread> + +#if defined(USE_NATPMP) || defined(USE_UPNP) +static CThreadInterrupt g_mapport_interrupt; +static std::thread g_mapport_thread; +static std::atomic_uint g_mapport_enabled_protos{MapPortProtoFlag::NONE}; +static std::atomic<MapPortProtoFlag> g_mapport_current_proto{MapPortProtoFlag::NONE}; + +using namespace std::chrono_literals; +static constexpr auto PORT_MAPPING_REANNOUNCE_PERIOD{20min}; +static constexpr auto PORT_MAPPING_RETRY_PERIOD{5min}; + +#ifdef USE_NATPMP +static uint16_t g_mapport_external_port = 0; +static bool NatpmpInit(natpmp_t* natpmp) +{ + const int r_init = initnatpmp(natpmp, /* detect gateway automatically */ 0, /* forced gateway - NOT APPLIED*/ 0); + if (r_init == 0) return true; + LogPrintf("natpmp: initnatpmp() failed with %d error.\n", r_init); + return false; +} + +static bool NatpmpDiscover(natpmp_t* natpmp, struct in_addr& external_ipv4_addr) +{ + const int r_send = sendpublicaddressrequest(natpmp); + if (r_send == 2 /* OK */) { + int r_read; + natpmpresp_t response; + do { + r_read = readnatpmpresponseorretry(natpmp, &response); + } while (r_read == NATPMP_TRYAGAIN); + + if (r_read == 0) { + external_ipv4_addr = response.pnu.publicaddress.addr; + return true; + } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { + LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); + } else { + LogPrintf("natpmp: readnatpmpresponseorretry() for public address failed with %d error.\n", r_read); + } + } else { + LogPrintf("natpmp: sendpublicaddressrequest() failed with %d error.\n", r_send); + } + + return false; +} + +static bool NatpmpMapping(natpmp_t* natpmp, const struct in_addr& external_ipv4_addr, uint16_t private_port, bool& external_ip_discovered) +{ + const uint16_t suggested_external_port = g_mapport_external_port ? g_mapport_external_port : private_port; + const int r_send = sendnewportmappingrequest(natpmp, NATPMP_PROTOCOL_TCP, private_port, suggested_external_port, 3600 /*seconds*/); + if (r_send == 12 /* OK */) { + int r_read; + natpmpresp_t response; + do { + r_read = readnatpmpresponseorretry(natpmp, &response); + } while (r_read == NATPMP_TRYAGAIN); + + if (r_read == 0) { + auto pm = response.pnu.newportmapping; + if (private_port == pm.privateport && pm.lifetime > 0) { + g_mapport_external_port = pm.mappedpublicport; + const CService external{external_ipv4_addr, pm.mappedpublicport}; + if (!external_ip_discovered && fDiscover) { + AddLocal(external, LOCAL_MAPPED); + external_ip_discovered = true; + } + LogPrintf("natpmp: Port mapping successful. External address = %s\n", external.ToString()); + return true; + } else { + LogPrintf("natpmp: Port mapping failed.\n"); + } + } else if (r_read == NATPMP_ERR_NOGATEWAYSUPPORT) { + LogPrintf("natpmp: The gateway does not support NAT-PMP.\n"); + } else { + LogPrintf("natpmp: readnatpmpresponseorretry() for port mapping failed with %d error.\n", r_read); + } + } else { + LogPrintf("natpmp: sendnewportmappingrequest() failed with %d error.\n", r_send); + } + + return false; +} + +static bool ProcessNatpmp() +{ + bool ret = false; + natpmp_t natpmp; + struct in_addr external_ipv4_addr; + if (NatpmpInit(&natpmp) && NatpmpDiscover(&natpmp, external_ipv4_addr)) { + bool external_ip_discovered = false; + const uint16_t private_port = GetListenPort(); + do { + ret = NatpmpMapping(&natpmp, external_ipv4_addr, private_port, external_ip_discovered); + } while (ret && g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); + g_mapport_interrupt.reset(); + + const int r_send = sendnewportmappingrequest(&natpmp, NATPMP_PROTOCOL_TCP, private_port, g_mapport_external_port, /* remove a port mapping */ 0); + g_mapport_external_port = 0; + if (r_send == 12 /* OK */) { + LogPrintf("natpmp: Port mapping removed successfully.\n"); + } else { + LogPrintf("natpmp: sendnewportmappingrequest(0) failed with %d error.\n", r_send); + } + } + + closenatpmp(&natpmp); + return ret; +} +#endif // USE_NATPMP + +#ifdef USE_UPNP +static bool ProcessUpnp() +{ + bool ret = false; + std::string port = strprintf("%u", GetListenPort()); + const char * multicastif = nullptr; + const char * minissdpdpath = nullptr; + struct UPNPDev * devlist = nullptr; + char lanaddr[64]; + + int error = 0; +#if MINIUPNPC_API_VERSION < 14 + devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, &error); +#else + devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error); +#endif + + struct UPNPUrls urls; + struct IGDdatas data; + int r; + + r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)); + if (r == 1) + { + if (fDiscover) { + char externalIPAddress[40]; + r = UPNP_GetExternalIPAddress(urls.controlURL, data.first.servicetype, externalIPAddress); + if (r != UPNPCOMMAND_SUCCESS) { + LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); + } else { + if (externalIPAddress[0]) { + CNetAddr resolved; + if (LookupHost(externalIPAddress, resolved, false)) { + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString()); + AddLocal(resolved, LOCAL_MAPPED); + } + } else { + LogPrintf("UPnP: GetExternalIPAddress failed.\n"); + } + } + } + + std::string strDesc = PACKAGE_NAME " " + FormatFullVersion(); + + do { + r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0"); + + if (r != UPNPCOMMAND_SUCCESS) { + ret = false; + LogPrintf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", port, port, lanaddr, r, strupnperror(r)); + break; + } else { + ret = true; + LogPrintf("UPnP Port Mapping successful.\n"); + } + } while (g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); + g_mapport_interrupt.reset(); + + r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0); + LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r); + freeUPNPDevlist(devlist); devlist = nullptr; + FreeUPNPUrls(&urls); + } else { + LogPrintf("No valid UPnP IGDs found\n"); + freeUPNPDevlist(devlist); devlist = nullptr; + if (r != 0) + FreeUPNPUrls(&urls); + } + + return ret; +} +#endif // USE_UPNP + +static void ThreadMapPort() +{ + bool ok; + do { + ok = false; + +#ifdef USE_UPNP + // High priority protocol. + if (g_mapport_enabled_protos & MapPortProtoFlag::UPNP) { + g_mapport_current_proto = MapPortProtoFlag::UPNP; + ok = ProcessUpnp(); + if (ok) continue; + } +#endif // USE_UPNP + +#ifdef USE_NATPMP + // Low priority protocol. + if (g_mapport_enabled_protos & MapPortProtoFlag::NAT_PMP) { + g_mapport_current_proto = MapPortProtoFlag::NAT_PMP; + ok = ProcessNatpmp(); + if (ok) continue; + } +#endif // USE_NATPMP + + g_mapport_current_proto = MapPortProtoFlag::NONE; + if (g_mapport_enabled_protos == MapPortProtoFlag::NONE) { + return; + } + + } while (ok || g_mapport_interrupt.sleep_for(PORT_MAPPING_RETRY_PERIOD)); +} + +void StartThreadMapPort() +{ + if (!g_mapport_thread.joinable()) { + assert(!g_mapport_interrupt); + g_mapport_thread = std::thread(std::bind(&TraceThread<void (*)()>, "mapport", &ThreadMapPort)); + } +} + +static void DispatchMapPort() +{ + if (g_mapport_current_proto == MapPortProtoFlag::NONE && g_mapport_enabled_protos == MapPortProtoFlag::NONE) { + return; + } + + if (g_mapport_current_proto == MapPortProtoFlag::NONE && g_mapport_enabled_protos != MapPortProtoFlag::NONE) { + StartThreadMapPort(); + return; + } + + if (g_mapport_current_proto != MapPortProtoFlag::NONE && g_mapport_enabled_protos == MapPortProtoFlag::NONE) { + InterruptMapPort(); + StopMapPort(); + return; + } + + if (g_mapport_enabled_protos & g_mapport_current_proto) { + // Enabling another protocol does not cause switching from the currently used one. + return; + } + + assert(g_mapport_thread.joinable()); + assert(!g_mapport_interrupt); + // Interrupt a protocol-specific loop in the ThreadUpnp() or in the ThreadNatpmp() + // to force trying the next protocol in the ThreadMapPort() loop. + g_mapport_interrupt(); +} + +static void MapPortProtoSetEnabled(MapPortProtoFlag proto, bool enabled) +{ + if (enabled) { + g_mapport_enabled_protos |= proto; + } else { + g_mapport_enabled_protos &= ~proto; + } +} + +void StartMapPort(bool use_upnp, bool use_natpmp) +{ + MapPortProtoSetEnabled(MapPortProtoFlag::UPNP, use_upnp); + MapPortProtoSetEnabled(MapPortProtoFlag::NAT_PMP, use_natpmp); + DispatchMapPort(); +} + +void InterruptMapPort() +{ + g_mapport_enabled_protos = MapPortProtoFlag::NONE; + if (g_mapport_thread.joinable()) { + g_mapport_interrupt(); + } +} + +void StopMapPort() +{ + if (g_mapport_thread.joinable()) { + g_mapport_thread.join(); + g_mapport_interrupt.reset(); + } +} + +#else // #if defined(USE_NATPMP) || defined(USE_UPNP) +void StartMapPort(bool use_upnp, bool use_natpmp) +{ + // Intentionally left blank. +} +void InterruptMapPort() +{ + // Intentionally left blank. +} +void StopMapPort() +{ + // Intentionally left blank. +} +#endif // #if defined(USE_NATPMP) || defined(USE_UPNP) diff --git a/src/mapport.h b/src/mapport.h new file mode 100644 index 0000000000..279d65167f --- /dev/null +++ b/src/mapport.h @@ -0,0 +1,30 @@ +// Copyright (c) 2011-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_MAPPORT_H +#define BITCOIN_MAPPORT_H + +#ifdef USE_UPNP +static constexpr bool DEFAULT_UPNP = USE_UPNP; +#else +static constexpr bool DEFAULT_UPNP = false; +#endif // USE_UPNP + +#ifdef USE_NATPMP +static constexpr bool DEFAULT_NATPMP = USE_NATPMP; +#else +static constexpr bool DEFAULT_NATPMP = false; +#endif // USE_NATPMP + +enum MapPortProtoFlag : unsigned int { + NONE = 0x00, + UPNP = 0x01, + NAT_PMP = 0x02, +}; + +void StartMapPort(bool use_upnp, bool use_natpmp); +void InterruptMapPort(); +void StopMapPort(); + +#endif // BITCOIN_MAPPORT_H diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index b571d463c9..3ffe1465da 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -59,7 +59,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve //if we do not have this assert, we can hit a memory access violation when indexing into vTxid assert(vTxid.size() != 0); if (height == 0) { - // hash at height 0 is the txids themself + // hash at height 0 is the txids themselves return vTxid[pos]; } else { // calculate left hash diff --git a/src/merkleblock.h b/src/merkleblock.h index b2d2828784..0e4ed72130 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 41a835f70a..009ae6b13a 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -213,7 +213,7 @@ bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost // - transaction finality (locktime) // - premature witness (in case segwit transactions are added to mempool before // segwit activation) -bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& package) +bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& package) const { for (CTxMemPool::txiter it : package) { if (!IsFinalTx(it->GetTx(), nHeight, nLockTimeCutoff)) diff --git a/src/miner.h b/src/miner.h index bb7a30b184..9a2b7063f4 100644 --- a/src/miner.h +++ b/src/miner.h @@ -185,7 +185,7 @@ private: * locktime, premature-witness, serialized size (if necessary) * These checks should always succeed, and they're here * only as an extra check in case of suboptimal node configuration */ - bool TestPackageTransactions(const CTxMemPool::setEntries& package); + bool TestPackageTransactions(const CTxMemPool::setEntries& package) const; /** Return true if given transaction from mapTx has already been evaluated, * or if the transaction's cached data in mapTx is incorrect. */ bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set& mapModifiedTx, CTxMemPool::setEntries& failedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); diff --git a/src/net.cpp b/src/net.cpp index cf987b6995..4f74bbede4 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <net_permissions.h> #include <netbase.h> #include <node/ui_interface.h> +#include <optional.h> #include <protocol.h> #include <random.h> #include <scheduler.h> @@ -32,15 +33,6 @@ #include <poll.h> #endif -#ifdef USE_UPNP -#include <miniupnpc/miniupnpc.h> -#include <miniupnpc/upnpcommands.h> -#include <miniupnpc/upnperrors.h> -// The minimum supported miniUPnPc API version is set to 10. This keeps compatibility -// with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -static_assert(MINIUPNPC_API_VERSION >= 10, "miniUPnPc API version >= 10 assumed"); -#endif - #include <algorithm> #include <cstdint> #include <unordered_map> @@ -72,6 +64,9 @@ static constexpr std::chrono::seconds DNSSEEDS_DELAY_FEW_PEERS{11}; static constexpr std::chrono::minutes DNSSEEDS_DELAY_MANY_PEERS{5}; static constexpr int DNSSEEDS_DELAY_PEER_THRESHOLD = 1000; // "many" vs "few" peers +/** The default timeframe for -maxuploadtarget. 1 day. */ +static constexpr std::chrono::seconds MAX_UPLOAD_TIMEFRAME{60 * 60 * 24}; + // We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. #define FEELER_SLEEP_WINDOW 1 @@ -111,7 +106,6 @@ static const uint64_t RANDOMIZER_ID_ADDRCACHE = 0x1cf2e4ddd306dda9ULL; // SHA256 // bool fDiscover = true; bool fListen = true; -bool g_relay_txes = !DEFAULT_BLOCKSONLY; RecursiveMutex cs_mapLocalHost; std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; @@ -478,7 +472,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); + CNode* pnode = new CNode(id, nLocalServices, hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); pnode->AddRef(); // We're making a new connection, harvest entropy from the time (and our peer count) @@ -504,9 +498,9 @@ void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNet } } -std::string CNode::ConnectionTypeAsString() const +std::string ConnectionTypeAsString(ConnectionType conn_type) { - switch (m_conn_type) { + switch (conn_type) { case ConnectionType::INBOUND: return "inbound"; case ConnectionType::MANUAL: @@ -552,7 +546,7 @@ void CNode::SetAddrLocal(const CService& addrLocalIn) { Network CNode::ConnectedThroughNetwork() const { - return IsInboundConn() && m_inbound_onion ? NET_ONION : addr.GetNetClass(); + return m_inbound_onion ? NET_ONION : addr.GetNetClass(); } #undef X @@ -563,7 +557,7 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) X(nServices); X(addr); X(addrBind); - stats.m_network = GetNetworkName(ConnectedThroughNetwork()); + stats.m_network = ConnectedThroughNetwork(); stats.m_mapped_as = addr.GetMappedAS(m_asmap); if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_filter); @@ -584,8 +578,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) X(cleanSubVer); } stats.fInbound = IsInboundConn(); - stats.m_manual_connection = IsManualConn(); - X(nStartingHeight); + X(m_bip152_highbandwidth_to); + X(m_bip152_highbandwidth_from); { LOCK(cs_vSend); X(mapSendBytesPerMsgCmd); @@ -596,10 +590,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) X(mapRecvBytesPerMsgCmd); X(nRecvBytes); } - X(m_legacyWhitelisted); X(m_permissionFlags); if (m_tx_relay != nullptr) { - LOCK(m_tx_relay->cs_feeFilter); stats.minFeeFilter = m_tx_relay->minFeeFilter; } else { stats.minFeeFilter = 0; @@ -625,38 +617,25 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) CService addrLocalUnlocked = GetAddrLocal(); stats.addrLocal = addrLocalUnlocked.IsValid() ? addrLocalUnlocked.ToString() : ""; - stats.m_conn_type_string = ConnectionTypeAsString(); + X(m_conn_type); } #undef X -/** - * Receive bytes from the buffer and deserialize them into messages. - * - * @param[in] pch A pointer to the raw data - * @param[in] nBytes Size of the data - * @param[out] complete Set True if at least one message has been - * deserialized and is ready to be processed - * @return True if the peer should stay connected, - * False if the peer should be disconnected from. - */ -bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete) +bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) { complete = false; const auto time = GetTime<std::chrono::microseconds>(); LOCK(cs_vRecv); nLastRecv = std::chrono::duration_cast<std::chrono::seconds>(time).count(); - nRecvBytes += nBytes; - while (nBytes > 0) { + nRecvBytes += msg_bytes.size(); + while (msg_bytes.size() > 0) { // absorb network data - int handled = m_deserializer->Read(pch, nBytes); + int handled = m_deserializer->Read(msg_bytes); if (handled < 0) { // Serious header problem, disconnect from the peer. return false; } - pch += handled; - nBytes -= handled; - if (m_deserializer->Complete()) { // decompose a transport agnostic CNetMessage from the deserializer uint32_t out_err_raw_size{0}; @@ -686,13 +665,13 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete return true; } -int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) { // copy data to temporary parsing buffer unsigned int nRemaining = CMessageHeader::HEADER_SIZE - nHdrPos; - unsigned int nCopy = std::min(nRemaining, nBytes); + unsigned int nCopy = std::min<unsigned int>(nRemaining, msg_bytes.size()); - memcpy(&hdrbuf[nHdrPos], pch, nCopy); + memcpy(&hdrbuf[nHdrPos], msg_bytes.data(), nCopy); nHdrPos += nCopy; // if header incomplete, exit @@ -726,18 +705,18 @@ int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) return nCopy; } -int V1TransportDeserializer::readData(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readData(Span<const uint8_t> msg_bytes) { unsigned int nRemaining = hdr.nMessageSize - nDataPos; - unsigned int nCopy = std::min(nRemaining, nBytes); + unsigned int nCopy = std::min<unsigned int>(nRemaining, msg_bytes.size()); if (vRecv.size() < nDataPos + nCopy) { // Allocate up to 256 KiB ahead, but never more than the total message size. vRecv.resize(std::min(hdr.nMessageSize, nDataPos + nCopy + 256 * 1024)); } - hasher.Write({(const unsigned char*)pch, nCopy}); - memcpy(&vRecv[nDataPos], pch, nCopy); + hasher.Write(msg_bytes.first(nCopy)); + memcpy(&vRecv[nDataPos], msg_bytes.data(), nCopy); nDataPos += nCopy; return nCopy; @@ -801,30 +780,30 @@ void V1TransportSerializer::prepareForTransport(CSerializedNetMsg& msg, std::vec CVectorWriter{SER_NETWORK, INIT_PROTO_VERSION, header, 0, hdr}; } -size_t CConnman::SocketSendData(CNode *pnode) const EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_vSend) +size_t CConnman::SocketSendData(CNode& node) const { - auto it = pnode->vSendMsg.begin(); + auto it = node.vSendMsg.begin(); size_t nSentSize = 0; - while (it != pnode->vSendMsg.end()) { - const auto &data = *it; - assert(data.size() > pnode->nSendOffset); + while (it != node.vSendMsg.end()) { + const auto& data = *it; + assert(data.size() > node.nSendOffset); int nBytes = 0; { - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) + LOCK(node.cs_hSocket); + if (node.hSocket == INVALID_SOCKET) break; - nBytes = send(pnode->hSocket, reinterpret_cast<const char*>(data.data()) + pnode->nSendOffset, data.size() - pnode->nSendOffset, MSG_NOSIGNAL | MSG_DONTWAIT); + nBytes = send(node.hSocket, reinterpret_cast<const char*>(data.data()) + node.nSendOffset, data.size() - node.nSendOffset, MSG_NOSIGNAL | MSG_DONTWAIT); } if (nBytes > 0) { - pnode->nLastSend = GetSystemTimeInSeconds(); - pnode->nSendBytes += nBytes; - pnode->nSendOffset += nBytes; + node.nLastSend = GetSystemTimeInSeconds(); + node.nSendBytes += nBytes; + node.nSendOffset += nBytes; nSentSize += nBytes; - if (pnode->nSendOffset == data.size()) { - pnode->nSendOffset = 0; - pnode->nSendSize -= data.size(); - pnode->fPauseSend = pnode->nSendSize > nSendBufferMaxSize; + if (node.nSendOffset == data.size()) { + node.nSendOffset = 0; + node.nSendSize -= data.size(); + node.fPauseSend = node.nSendSize > nSendBufferMaxSize; it++; } else { // could not send full message; stop sending more @@ -834,10 +813,9 @@ size_t CConnman::SocketSendData(CNode *pnode) const EXCLUSIVE_LOCKS_REQUIRED(pno if (nBytes < 0) { // error int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) - { + if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { LogPrintf("socket send error %s\n", NetworkErrorString(nErr)); - pnode->CloseSocketDisconnect(); + node.CloseSocketDisconnect(); } } // couldn't send anything at all @@ -845,29 +823,14 @@ size_t CConnman::SocketSendData(CNode *pnode) const EXCLUSIVE_LOCKS_REQUIRED(pno } } - if (it == pnode->vSendMsg.end()) { - assert(pnode->nSendOffset == 0); - assert(pnode->nSendSize == 0); + if (it == node.vSendMsg.end()) { + assert(node.nSendOffset == 0); + assert(node.nSendSize == 0); } - pnode->vSendMsg.erase(pnode->vSendMsg.begin(), it); + node.vSendMsg.erase(node.vSendMsg.begin(), it); return nSentSize; } -struct NodeEvictionCandidate -{ - NodeId id; - int64_t nTimeConnected; - int64_t nMinPingUsecTime; - int64_t nLastBlockTime; - int64_t nLastTXTime; - bool fRelevantServices; - bool fRelayTxes; - bool fBloomFilter; - uint64_t nKeyedNetGroup; - bool prefer_evict; - bool m_is_local; -}; - static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { return a.nMinPingUsecTime > b.nMinPingUsecTime; @@ -923,43 +886,8 @@ static void EraseLastKElements(std::vector<T> &elements, Comparator comparator, elements.erase(elements.end() - eraseSize, elements.end()); } -/** Try to find a connection to evict when the node is full. - * Extreme care must be taken to avoid opening the node to attacker - * triggered network partitioning. - * The strategy used here is to protect a small number of peers - * for each of several distinct characteristics which are difficult - * to forge. In order to partition a node the attacker must be - * simultaneously better at all of them than honest peers. - */ -bool CConnman::AttemptToEvictConnection() +[[nodiscard]] Optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates) { - std::vector<NodeEvictionCandidate> vEvictionCandidates; - { - LOCK(cs_vNodes); - - for (const CNode* node : vNodes) { - if (node->HasPermission(PF_NOBAN)) - continue; - if (!node->IsInboundConn()) - continue; - if (node->fDisconnect) - continue; - bool peer_relay_txes = false; - bool peer_filter_not_null = false; - if (node->m_tx_relay != nullptr) { - LOCK(node->m_tx_relay->cs_filter); - peer_relay_txes = node->m_tx_relay->fRelayTxes; - peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; - } - NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, - node->nLastBlockTime, node->nLastTXTime, - HasAllDesirableServiceFlags(node->nServices), - peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, - node->m_prefer_evict, node->addr.IsLocal()}; - vEvictionCandidates.push_back(candidate); - } - } - // Protect connections with certain characteristics // Deterministically select 4 peers to protect by netgroup. @@ -997,7 +925,7 @@ bool CConnman::AttemptToEvictConnection() total_protect_size -= initial_size - vEvictionCandidates.size(); EraseLastKElements(vEvictionCandidates, ReverseCompareNodeTimeConnected, total_protect_size); - if (vEvictionCandidates.empty()) return false; + if (vEvictionCandidates.empty()) return nullopt; // If any remaining peers are preferred for eviction consider only them. // This happens after the other preferences since if a peer is really the best by other criteria (esp relaying blocks) @@ -1029,10 +957,52 @@ bool CConnman::AttemptToEvictConnection() vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); // Disconnect from the network group with the most connections - NodeId evicted = vEvictionCandidates.front().id; + return vEvictionCandidates.front().id; +} + +/** Try to find a connection to evict when the node is full. + * Extreme care must be taken to avoid opening the node to attacker + * triggered network partitioning. + * The strategy used here is to protect a small number of peers + * for each of several distinct characteristics which are difficult + * to forge. In order to partition a node the attacker must be + * simultaneously better at all of them than honest peers. + */ +bool CConnman::AttemptToEvictConnection() +{ + std::vector<NodeEvictionCandidate> vEvictionCandidates; + { + + LOCK(cs_vNodes); + for (const CNode* node : vNodes) { + if (node->HasPermission(PF_NOBAN)) + continue; + if (!node->IsInboundConn()) + continue; + if (node->fDisconnect) + continue; + bool peer_relay_txes = false; + bool peer_filter_not_null = false; + if (node->m_tx_relay != nullptr) { + LOCK(node->m_tx_relay->cs_filter); + peer_relay_txes = node->m_tx_relay->fRelayTxes; + peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; + } + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, + node->nLastBlockTime, node->nLastTXTime, + HasAllDesirableServiceFlags(node->nServices), + peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, + node->m_prefer_evict, node->addr.IsLocal()}; + vEvictionCandidates.push_back(candidate); + } + } + const Optional<NodeId> node_id_to_evict = SelectNodeToEvict(std::move(vEvictionCandidates)); + if (!node_id_to_evict) { + return false; + } LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (pnode->GetId() == evicted) { + if (pnode->GetId() == *node_id_to_evict) { pnode->fDisconnect = true; return true; } @@ -1057,14 +1027,12 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { NetPermissionFlags permissionFlags = NetPermissionFlags::PF_NONE; hListenSocket.AddSocketPermissionFlags(permissionFlags); AddWhitelistPermissionFlags(permissionFlags, addr); - bool legacyWhitelisted = false; if (NetPermissions::HasFlag(permissionFlags, NetPermissionFlags::PF_ISIMPLICIT)) { NetPermissions::ClearFlag(permissionFlags, PF_ISIMPLICIT); if (gArgs.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) NetPermissions::AddFlag(permissionFlags, PF_FORCERELAY); if (gArgs.GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) NetPermissions::AddFlag(permissionFlags, PF_RELAY); NetPermissions::AddFlag(permissionFlags, PF_MEMPOOL); NetPermissions::AddFlag(permissionFlags, PF_NOBAN); - legacyWhitelisted = true; } { @@ -1137,11 +1105,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); - CNode* pnode = new CNode(id, nodeServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", ConnectionType::INBOUND, inbound_onion); + CNode* pnode = new CNode(id, nodeServices, hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", ConnectionType::INBOUND, inbound_onion); pnode->AddRef(); pnode->m_permissionFlags = permissionFlags; - // If this flag is present, the user probably expect that RPC and QT report it as whitelisted (backward compatibility) - pnode->m_legacyWhitelisted = legacyWhitelisted; pnode->m_prefer_evict = discouraged; m_msgproc->InitializeNode(pnode); @@ -1156,6 +1122,27 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { RandAddEvent((uint32_t)id); } +bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type) +{ + if (conn_type != ConnectionType::OUTBOUND_FULL_RELAY && conn_type != ConnectionType::BLOCK_RELAY) return false; + + const int max_connections = conn_type == ConnectionType::OUTBOUND_FULL_RELAY ? m_max_outbound_full_relay : m_max_outbound_block_relay; + + // Count existing connections + int existing_connections = WITH_LOCK(cs_vNodes, + return std::count_if(vNodes.begin(), vNodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); + + // Max connections of specified type already exist + if (existing_connections >= max_connections) return false; + + // Max total outbound connections already exist + CSemaphoreGrant grant(*semOutbound, true); + if (!grant) return false; + + OpenNetworkConnection(CAddress(), false, &grant, address.c_str(), conn_type); + return true; +} + void CConnman::DisconnectNodes() { { @@ -1229,7 +1216,7 @@ void CConnman::NotifyNumConnectionsChanged() } } -void CConnman::InactivityCheck(CNode *pnode) +void CConnman::InactivityCheck(CNode *pnode) const { int64_t nTime = GetSystemTimeInSeconds(); if (nTime - pnode->nTimeConnected > m_peer_connect_timeout) @@ -1244,7 +1231,7 @@ void CConnman::InactivityCheck(CNode *pnode) LogPrintf("socket sending timeout: %is\n", nTime - pnode->nLastSend); pnode->fDisconnect = true; } - else if (nTime - pnode->nLastRecv > (pnode->GetCommonVersion() > BIP0031_VERSION ? TIMEOUT_INTERVAL : 90*60)) + else if (nTime - pnode->nLastRecv > TIMEOUT_INTERVAL) { LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); pnode->fDisconnect = true; @@ -1476,18 +1463,18 @@ void CConnman::SocketHandler() if (recvSet || errorSet) { // typical socket buffer is 8K-64K - char pchBuf[0x10000]; + uint8_t pchBuf[0x10000]; int nBytes = 0; { LOCK(pnode->cs_hSocket); if (pnode->hSocket == INVALID_SOCKET) continue; - nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); + nBytes = recv(pnode->hSocket, (char*)pchBuf, sizeof(pchBuf), MSG_DONTWAIT); } if (nBytes > 0) { bool notify = false; - if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) + if (!pnode->ReceiveMsgBytes(Span<const uint8_t>(pchBuf, nBytes), notify)) pnode->CloseSocketDisconnect(); RecordBytesRecv(nBytes); if (notify) { @@ -1529,16 +1516,10 @@ void CConnman::SocketHandler() } } - // - // Send - // - if (sendSet) - { - LOCK(pnode->cs_vSend); - size_t nBytes = SocketSendData(pnode); - if (nBytes) { - RecordBytesSent(nBytes); - } + if (sendSet) { + // Send data + size_t bytes_sent = WITH_LOCK(pnode->cs_vSend, return SocketSendData(*pnode)); + if (bytes_sent) RecordBytesSent(bytes_sent); } InactivityCheck(pnode); @@ -1569,121 +1550,6 @@ void CConnman::WakeMessageHandler() condMsgProc.notify_one(); } - - - - - -#ifdef USE_UPNP -static CThreadInterrupt g_upnp_interrupt; -static std::thread g_upnp_thread; -static void ThreadMapPort() -{ - std::string port = strprintf("%u", GetListenPort()); - const char * multicastif = nullptr; - const char * minissdpdpath = nullptr; - struct UPNPDev * devlist = nullptr; - char lanaddr[64]; - - int error = 0; -#if MINIUPNPC_API_VERSION < 14 - devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, &error); -#else - devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error); -#endif - - struct UPNPUrls urls; - struct IGDdatas data; - int r; - - r = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)); - if (r == 1) - { - if (fDiscover) { - char externalIPAddress[40]; - r = UPNP_GetExternalIPAddress(urls.controlURL, data.first.servicetype, externalIPAddress); - if (r != UPNPCOMMAND_SUCCESS) { - LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); - } else { - if (externalIPAddress[0]) { - CNetAddr resolved; - if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString()); - AddLocal(resolved, LOCAL_UPNP); - } - } else { - LogPrintf("UPnP: GetExternalIPAddress failed.\n"); - } - } - } - - std::string strDesc = PACKAGE_NAME " " + FormatFullVersion(); - - do { - r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0"); - - if (r != UPNPCOMMAND_SUCCESS) { - LogPrintf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", port, port, lanaddr, r, strupnperror(r)); - } else { - LogPrintf("UPnP Port Mapping successful.\n"); - } - } while (g_upnp_interrupt.sleep_for(std::chrono::minutes(20))); - - r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0); - LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r); - freeUPNPDevlist(devlist); devlist = nullptr; - FreeUPNPUrls(&urls); - } else { - LogPrintf("No valid UPnP IGDs found\n"); - freeUPNPDevlist(devlist); devlist = nullptr; - if (r != 0) - FreeUPNPUrls(&urls); - } -} - -void StartMapPort() -{ - if (!g_upnp_thread.joinable()) { - assert(!g_upnp_interrupt); - g_upnp_thread = std::thread((std::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort))); - } -} - -void InterruptMapPort() -{ - if(g_upnp_thread.joinable()) { - g_upnp_interrupt(); - } -} - -void StopMapPort() -{ - if(g_upnp_thread.joinable()) { - g_upnp_thread.join(); - g_upnp_interrupt.reset(); - } -} - -#else -void StartMapPort() -{ - // Intentionally left blank. -} -void InterruptMapPort() -{ - // Intentionally left blank. -} -void StopMapPort() -{ - // Intentionally left blank. -} -#endif - - - - - - void CConnman::ThreadDNSAddressSeed() { FastRandomContext rng; @@ -1839,18 +1705,32 @@ void CConnman::SetTryNewOutboundPeer(bool flag) // Also exclude peers that haven't finished initial connection handshake yet // (so that we don't decide we're over our desired connection limit, and then // evict some peer that has finished the handshake) -int CConnman::GetExtraOutboundCount() +int CConnman::GetExtraFullOutboundCount() +{ + int full_outbound_peers = 0; + { + LOCK(cs_vNodes); + for (const CNode* pnode : vNodes) { + if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsFullOutboundConn()) { + ++full_outbound_peers; + } + } + } + return std::max(full_outbound_peers - m_max_outbound_full_relay, 0); +} + +int CConnman::GetExtraBlockRelayCount() { - int nOutbound = 0; + int block_relay_peers = 0; { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsOutboundOrBlockRelayConn()) { - ++nOutbound; + if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsBlockOnlyConn()) { + ++block_relay_peers; } } } - return std::max(nOutbound - m_max_outbound_full_relay - m_max_outbound_block_relay, 0); + return std::max(block_relay_peers - m_max_outbound_block_relay, 0); } void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) @@ -1881,6 +1761,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Minimum time before next feeler connection (in microseconds). int64_t nNextFeeler = PoissonNextSend(nStart*1000*1000, FEELER_INTERVAL); + int64_t nNextExtraBlockRelay = PoissonNextSend(nStart*1000*1000, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); while (!interruptNet) { ProcessAddrFetch(); @@ -1953,8 +1834,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // until we hit our block-relay-only peer limit. // GetTryNewOutboundPeer() gets set when a stale tip is detected, so we // try opening an additional OUTBOUND_FULL_RELAY connection. If none of - // these conditions are met, check the nNextFeeler timer to decide if - // we should open a FEELER. + // these conditions are met, check to see if it's time to try an extra + // block-relay-only peer (to confirm our tip is current, see below) or the nNextFeeler + // timer to decide if we should open a FEELER. if (!m_anchors.empty() && (nOutboundBlockRelay < m_max_outbound_block_relay)) { conn_type = ConnectionType::BLOCK_RELAY; @@ -1965,6 +1847,30 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) conn_type = ConnectionType::BLOCK_RELAY; } else if (GetTryNewOutboundPeer()) { // OUTBOUND_FULL_RELAY + } else if (nTime > nNextExtraBlockRelay && m_start_extra_block_relay_peers) { + // Periodically connect to a peer (using regular outbound selection + // methodology from addrman) and stay connected long enough to sync + // headers, but not much else. + // + // Then disconnect the peer, if we haven't learned anything new. + // + // The idea is to make eclipse attacks very difficult to pull off, + // because every few minutes we're finding a new peer to learn headers + // from. + // + // This is similar to the logic for trying extra outbound (full-relay) + // peers, except: + // - we do this all the time on a poisson timer, rather than just when + // our tip is stale + // - we potentially disconnect our next-youngest block-relay-only peer, if our + // newest block-relay-only peer delivers a block more recently. + // See the eviction logic in net_processing.cpp. + // + // Because we can promote these connections to block-relay-only + // connections, they do not get their own ConnectionType enum + // (similar to how we deal with extra outbound peers). + nNextExtraBlockRelay = PoissonNextSend(nTime, EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL); + conn_type = ConnectionType::BLOCK_RELAY; } else if (nTime > nNextFeeler) { nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); conn_type = ConnectionType::FEELER; @@ -2050,7 +1956,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; } - // do not allow non-default ports, unless after 50 invalid addresses selected already + // Do not allow non-default ports, unless after 50 invalid + // addresses selected already. This is to prevent malicious peers + // from advertising themselves as a service on another host and + // port, causing a DoS attack as nodes around the network attempt + // to connect to it fruitlessly. if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; @@ -2445,17 +2355,6 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { Init(connOptions); - { - LOCK(cs_totalBytesRecv); - nTotalBytesRecv = 0; - } - { - LOCK(cs_totalBytesSent); - nTotalBytesSent = 0; - nMaxOutboundTotalBytesSentInCycle = 0; - nMaxOutboundCycleStartTime = 0; - } - if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds, connOptions.onion_binds)) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( @@ -2830,8 +2729,8 @@ void CConnman::RecordBytesSent(uint64_t bytes) LOCK(cs_totalBytesSent); nTotalBytesSent += bytes; - uint64_t now = GetTime(); - if (nMaxOutboundCycleStartTime + nMaxOutboundTimeframe < now) + const auto now = GetTime<std::chrono::seconds>(); + if (nMaxOutboundCycleStartTime + MAX_UPLOAD_TIMEFRAME < now) { // timeframe expired, reset cycle nMaxOutboundCycleStartTime = now; @@ -2842,48 +2741,29 @@ void CConnman::RecordBytesSent(uint64_t bytes) nMaxOutboundTotalBytesSentInCycle += bytes; } -void CConnman::SetMaxOutboundTarget(uint64_t limit) -{ - LOCK(cs_totalBytesSent); - nMaxOutboundLimit = limit; -} - uint64_t CConnman::GetMaxOutboundTarget() { LOCK(cs_totalBytesSent); return nMaxOutboundLimit; } -uint64_t CConnman::GetMaxOutboundTimeframe() +std::chrono::seconds CConnman::GetMaxOutboundTimeframe() { - LOCK(cs_totalBytesSent); - return nMaxOutboundTimeframe; + return MAX_UPLOAD_TIMEFRAME; } -uint64_t CConnman::GetMaxOutboundTimeLeftInCycle() +std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle() { LOCK(cs_totalBytesSent); if (nMaxOutboundLimit == 0) - return 0; - - if (nMaxOutboundCycleStartTime == 0) - return nMaxOutboundTimeframe; + return 0s; - uint64_t cycleEndTime = nMaxOutboundCycleStartTime + nMaxOutboundTimeframe; - uint64_t now = GetTime(); - return (cycleEndTime < now) ? 0 : cycleEndTime - GetTime(); -} + if (nMaxOutboundCycleStartTime.count() == 0) + return MAX_UPLOAD_TIMEFRAME; -void CConnman::SetMaxOutboundTimeframe(uint64_t timeframe) -{ - LOCK(cs_totalBytesSent); - if (nMaxOutboundTimeframe != timeframe) - { - // reset measure-cycle in case of changing - // the timeframe - nMaxOutboundCycleStartTime = GetTime(); - } - nMaxOutboundTimeframe = timeframe; + const std::chrono::seconds cycleEndTime = nMaxOutboundCycleStartTime + MAX_UPLOAD_TIMEFRAME; + const auto now = GetTime<std::chrono::seconds>(); + return (cycleEndTime < now) ? 0s : cycleEndTime - now; } bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) @@ -2895,8 +2775,8 @@ bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) if (historicalBlockServingLimit) { // keep a large enough buffer to at least relay each block once - uint64_t timeLeftInCycle = GetMaxOutboundTimeLeftInCycle(); - uint64_t buffer = timeLeftInCycle / 600 * MAX_BLOCK_SERIALIZED_SIZE; + const std::chrono::seconds timeLeftInCycle = GetMaxOutboundTimeLeftInCycle(); + const uint64_t buffer = timeLeftInCycle / std::chrono::minutes{10} * MAX_BLOCK_SERIALIZED_SIZE; if (buffer >= nMaxOutboundLimit || nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit - buffer) return true; } @@ -2932,36 +2812,22 @@ ServiceFlags CConnman::GetLocalServices() const return nLocalServices; } -void CConnman::SetBestHeight(int height) -{ - nBestHeight.store(height, std::memory_order_release); -} - -int CConnman::GetBestHeight() const -{ - return nBestHeight.load(std::memory_order_acquire); -} - unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion) +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion) : nTimeConnected(GetSystemTimeInSeconds()), - addr(addrIn), - addrBind(addrBindIn), - nKeyedNetGroup(nKeyedNetGroupIn), - // Don't relay addr messages to peers that we connect to as block-relay-only - // peers (to prevent adversaries from inferring these links from addr - // traffic). - id(idIn), - nLocalHostNonce(nLocalHostNonceIn), - m_conn_type(conn_type_in), - nLocalServices(nLocalServicesIn), - nMyStartingHeight(nMyStartingHeightIn), - m_inbound_onion(inbound_onion) -{ + addr(addrIn), + addrBind(addrBindIn), + nKeyedNetGroup(nKeyedNetGroupIn), + id(idIn), + nLocalHostNonce(nLocalHostNonceIn), + m_conn_type(conn_type_in), + nLocalServices(nLocalServicesIn), + m_inbound_onion(inbound_onion) +{ + if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); hSocket = hSocketIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; - hashContinue = uint256(); if (conn_type_in != ConnectionType::BLOCK_RELAY) { m_tx_relay = MakeUnique<TxRelay>(); } @@ -3021,7 +2887,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) // If write queue empty, attempt "optimistic write" if (optimisticSend == true) - nBytesSent = SocketSendData(pnode); + nBytesSent = SocketSendData(*pnode); } if (nBytesSent) RecordBytesSent(nBytesSent); @@ -24,19 +24,16 @@ #include <sync.h> #include <threadinterrupt.h> #include <uint256.h> +#include <util/check.h> #include <atomic> +#include <condition_variable> #include <cstdint> #include <deque> #include <map> -#include <thread> #include <memory> -#include <condition_variable> - -#ifndef WIN32 -#include <arpa/inet.h> -#endif - +#include <thread> +#include <vector> class CScheduler; class CNode; @@ -52,6 +49,8 @@ static const bool DEFAULT_WHITELISTFORCERELAY = false; static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; +/** Run the extra block-relay-only connection loop once every 5 minutes. **/ +static const int EXTRA_BLOCK_RELAY_ONLY_PEER_INTERVAL = 300; /** The maximum number of addresses from our addrman to return in response to a getaddr message. */ static constexpr size_t MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ @@ -68,18 +67,10 @@ static const int MAX_BLOCK_RELAY_ONLY_CONNECTIONS = 2; static const int MAX_FEELER_CONNECTIONS = 1; /** -listen default */ static const bool DEFAULT_LISTEN = true; -/** -upnp default */ -#ifdef USE_UPNP -static const bool DEFAULT_UPNP = USE_UPNP; -#else -static const bool DEFAULT_UPNP = false; -#endif /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ -static const uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; -/** The default timeframe for -maxuploadtarget. 1 day. */ -static const uint64_t MAX_UPLOAD_TIMEFRAME = 60 * 60 * 24; +static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; /** Default for blocks only*/ static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ @@ -183,463 +174,17 @@ enum class ConnectionType { ADDR_FETCH, }; -class NetEventsInterface; -class CConnman -{ -public: - - enum NumConnections { - CONNECTIONS_NONE = 0, - CONNECTIONS_IN = (1U << 0), - CONNECTIONS_OUT = (1U << 1), - CONNECTIONS_ALL = (CONNECTIONS_IN | CONNECTIONS_OUT), - }; - - struct Options - { - ServiceFlags nLocalServices = NODE_NONE; - int nMaxConnections = 0; - int m_max_outbound_full_relay = 0; - int m_max_outbound_block_relay = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; - int nBestHeight = 0; - CClientUIInterface* uiInterface = nullptr; - NetEventsInterface* m_msgproc = nullptr; - BanMan* m_banman = nullptr; - unsigned int nSendBufferMaxSize = 0; - unsigned int nReceiveFloodSize = 0; - uint64_t nMaxOutboundTimeframe = 0; - uint64_t nMaxOutboundLimit = 0; - int64_t m_peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT; - std::vector<std::string> vSeedNodes; - std::vector<NetWhitelistPermissions> vWhitelistedRange; - std::vector<NetWhitebindPermissions> vWhiteBinds; - std::vector<CService> vBinds; - std::vector<CService> onion_binds; - bool m_use_addrman_outgoing = true; - std::vector<std::string> m_specified_outgoing; - std::vector<std::string> m_added_nodes; - std::vector<bool> m_asmap; - }; - - void Init(const Options& connOptions) { - nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); - m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; - m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; - m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; - nBestHeight = connOptions.nBestHeight; - clientInterface = connOptions.uiInterface; - m_banman = connOptions.m_banman; - m_msgproc = connOptions.m_msgproc; - nSendBufferMaxSize = connOptions.nSendBufferMaxSize; - nReceiveFloodSize = connOptions.nReceiveFloodSize; - m_peer_connect_timeout = connOptions.m_peer_connect_timeout; - { - LOCK(cs_totalBytesSent); - nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; - nMaxOutboundLimit = connOptions.nMaxOutboundLimit; - } - vWhitelistedRange = connOptions.vWhitelistedRange; - { - LOCK(cs_vAddedNodes); - vAddedNodes = connOptions.m_added_nodes; - } - m_onion_binds = connOptions.onion_binds; - } - - CConnman(uint64_t seed0, uint64_t seed1, bool network_active = true); - ~CConnman(); - bool Start(CScheduler& scheduler, const Options& options); - - void StopThreads(); - void StopNodes(); - void Stop() - { - StopThreads(); - StopNodes(); - }; - - void Interrupt(); - bool GetNetworkActive() const { return fNetworkActive; }; - bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; - void SetNetworkActive(bool active); - void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant* grantOutbound, const char* strDest, ConnectionType conn_type); - bool CheckIncomingNonce(uint64_t nonce); - - bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); - - void PushMessage(CNode* pnode, CSerializedNetMsg&& msg); - - using NodeFn = std::function<void(CNode*)>; - void ForEachNode(const NodeFn& func) - { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { - if (NodeFullyConnected(node)) - func(node); - } - }; - - void ForEachNode(const NodeFn& func) const - { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { - if (NodeFullyConnected(node)) - func(node); - } - }; - - template<typename Callable, typename CallableAfter> - void ForEachNodeThen(Callable&& pre, CallableAfter&& post) - { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { - if (NodeFullyConnected(node)) - pre(node); - } - post(); - }; - - template<typename Callable, typename CallableAfter> - void ForEachNodeThen(Callable&& pre, CallableAfter&& post) const - { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { - if (NodeFullyConnected(node)) - pre(node); - } - post(); - }; - - // Addrman functions - void SetServices(const CService &addr, ServiceFlags nServices); - void MarkAddressGood(const CAddress& addr); - bool AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); - std::vector<CAddress> GetAddresses(size_t max_addresses, size_t max_pct); - /** - * Cache is used to minimize topology leaks, so it should - * be used for all non-trusted calls, for example, p2p. - * A non-malicious call (from RPC or a peer with addr permission) should - * call the function without a parameter to avoid using the cache. - */ - std::vector<CAddress> GetAddresses(CNode& requestor, size_t max_addresses, size_t max_pct); - - // This allows temporarily exceeding m_max_outbound_full_relay, with the goal of finding - // a peer that is better than all our current peers. - void SetTryNewOutboundPeer(bool flag); - bool GetTryNewOutboundPeer(); - - // Return the number of outbound peers we have in excess of our target (eg, - // if we previously called SetTryNewOutboundPeer(true), and have since set - // to false, we may have extra peers that we wish to disconnect). This may - // return a value less than (num_outbound_connections - num_outbound_slots) - // in cases where some outbound connections are not yet fully connected, or - // not yet fully disconnected. - int GetExtraOutboundCount(); - - bool AddNode(const std::string& node); - bool RemoveAddedNode(const std::string& node); - std::vector<AddedNodeInfo> GetAddedNodeInfo(); - - size_t GetNodeCount(NumConnections num); - void GetNodeStats(std::vector<CNodeStats>& vstats); - bool DisconnectNode(const std::string& node); - bool DisconnectNode(const CSubNet& subnet); - bool DisconnectNode(const CNetAddr& addr); - bool DisconnectNode(NodeId id); - - //! Used to convey which local services we are offering peers during node - //! connection. - //! - //! The data returned by this is used in CNode construction, - //! which is used to advertise which services we are offering - //! that peer during `net_processing.cpp:PushNodeVersion()`. - ServiceFlags GetLocalServices() const; - - //!set the max outbound target in bytes - void SetMaxOutboundTarget(uint64_t limit); - uint64_t GetMaxOutboundTarget(); - - //!set the timeframe for the max outbound target - void SetMaxOutboundTimeframe(uint64_t timeframe); - uint64_t GetMaxOutboundTimeframe(); - - //! check if the outbound target is reached - //! if param historicalBlockServingLimit is set true, the function will - //! response true if the limit for serving historical blocks has been reached - bool OutboundTargetReached(bool historicalBlockServingLimit); - - //! response the bytes left in the current max outbound cycle - //! in case of no limit, it will always response 0 - uint64_t GetOutboundTargetBytesLeft(); - - //! response the time in second left in the current max outbound cycle - //! in case of no limit, it will always response 0 - uint64_t GetMaxOutboundTimeLeftInCycle(); - - uint64_t GetTotalBytesRecv(); - uint64_t GetTotalBytesSent(); - - void SetBestHeight(int height); - int GetBestHeight() const; - - /** Get a unique deterministic randomizer. */ - CSipHasher GetDeterministicRandomizer(uint64_t id) const; - - unsigned int GetReceiveFloodSize() const; - - void WakeMessageHandler(); - - /** Attempts to obfuscate tx time through exponentially distributed emitting. - Works assuming that a single interval is used. - Variable intervals will result in privacy decrease. - */ - int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); - - void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } - -private: - struct ListenSocket { - public: - SOCKET socket; - inline void AddSocketPermissionFlags(NetPermissionFlags& flags) const { NetPermissions::AddFlag(flags, m_permissions); } - ListenSocket(SOCKET socket_, NetPermissionFlags permissions_) : socket(socket_), m_permissions(permissions_) {} - private: - NetPermissionFlags m_permissions; - }; - - bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); - bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); - bool InitBinds( - const std::vector<CService>& binds, - const std::vector<NetWhitebindPermissions>& whiteBinds, - const std::vector<CService>& onion_binds); - - void ThreadOpenAddedConnections(); - void AddAddrFetch(const std::string& strDest); - void ProcessAddrFetch(); - void ThreadOpenConnections(std::vector<std::string> connect); - void ThreadMessageHandler(); - void AcceptConnection(const ListenSocket& hListenSocket); - void DisconnectNodes(); - void NotifyNumConnectionsChanged(); - void InactivityCheck(CNode *pnode); - bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); - void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); - void SocketHandler(); - void ThreadSocketHandler(); - void ThreadDNSAddressSeed(); - - uint64_t CalculateKeyedNetGroup(const CAddress& ad) const; - - CNode* FindNode(const CNetAddr& ip); - CNode* FindNode(const CSubNet& subNet); - CNode* FindNode(const std::string& addrName); - CNode* FindNode(const CService& addr); - - /** - * Determine whether we're already connected to a given address, in order to - * avoid initiating duplicate connections. - */ - bool AlreadyConnectedToAddress(const CAddress& addr); - - bool AttemptToEvictConnection(); - CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type); - void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const; - - void DeleteNode(CNode* pnode); - - NodeId GetNewNodeId(); - - size_t SocketSendData(CNode *pnode) const; - void DumpAddresses(); - - // Network stats - void RecordBytesRecv(uint64_t bytes); - void RecordBytesSent(uint64_t bytes); - - /** - * Return vector of current BLOCK_RELAY peers. - */ - std::vector<CAddress> GetCurrentBlockRelayOnlyConns() const; - - // Whether the node should be passed out in ForEach* callbacks - static bool NodeFullyConnected(const CNode* pnode); - - // Network usage totals - RecursiveMutex cs_totalBytesRecv; - RecursiveMutex cs_totalBytesSent; - uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; - uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; - - // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent); - uint64_t nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent); - uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); - uint64_t nMaxOutboundTimeframe GUARDED_BY(cs_totalBytesSent); - - // P2P timeout in seconds - int64_t m_peer_connect_timeout; - - // Whitelisted ranges. Any node connecting from these is automatically - // whitelisted (as well as those connecting to whitelisted binds). - std::vector<NetWhitelistPermissions> vWhitelistedRange; - - unsigned int nSendBufferMaxSize{0}; - unsigned int nReceiveFloodSize{0}; - - std::vector<ListenSocket> vhListenSocket; - std::atomic<bool> fNetworkActive{true}; - bool fAddressesInitialized{false}; - CAddrMan addrman; - std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); - RecursiveMutex m_addr_fetches_mutex; - std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); - RecursiveMutex cs_vAddedNodes; - std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); - std::list<CNode*> vNodesDisconnected; - mutable RecursiveMutex cs_vNodes; - std::atomic<NodeId> nLastNodeId{0}; - unsigned int nPrevNodeCount{0}; - - /** - * Cache responses to addr requests to minimize privacy leak. - * Attack example: scraping addrs in real-time may allow an attacker - * to infer new connections of the victim by detecting new records - * with fresh timestamps (per self-announcement). - */ - struct CachedAddrResponse { - std::vector<CAddress> m_addrs_response_cache; - std::chrono::microseconds m_cache_entry_expiration{0}; - }; - - /** - * Addr responses stored in different caches - * per (network, local socket) prevent cross-network node identification. - * If a node for example is multi-homed under Tor and IPv6, - * a single cache (or no cache at all) would let an attacker - * to easily detect that it is the same node by comparing responses. - * Indexing by local socket prevents leakage when a node has multiple - * listening addresses on the same network. - * - * The used memory equals to 1000 CAddress records (or around 40 bytes) per - * distinct Network (up to 5) we have/had an inbound peer from, - * resulting in at most ~196 KB. Every separate local socket may - * add up to ~196 KB extra. - */ - std::map<uint64_t, CachedAddrResponse> m_addr_response_caches; - - /** - * Services this instance offers. - * - * This data is replicated in each CNode instance we create during peer - * connection (in ConnectNode()) under a member also called - * nLocalServices. - * - * This data is not marked const, but after being set it should not - * change. See the note in CNode::nLocalServices documentation. - * - * \sa CNode::nLocalServices - */ - ServiceFlags nLocalServices; - - std::unique_ptr<CSemaphore> semOutbound; - std::unique_ptr<CSemaphore> semAddnode; - int nMaxConnections; - - // How many full-relay (tx, block, addr) outbound peers we want - int m_max_outbound_full_relay; - - // How many block-relay only outbound peers we want - // We do not relay tx or addr messages with these peers - int m_max_outbound_block_relay; - - int nMaxAddnode; - int nMaxFeeler; - int m_max_outbound; - bool m_use_addrman_outgoing; - std::atomic<int> nBestHeight; - CClientUIInterface* clientInterface; - NetEventsInterface* m_msgproc; - /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ - BanMan* m_banman; - - /** - * Addresses that were saved during the previous clean shutdown. We'll - * attempt to make block-relay-only connections to them. - */ - std::vector<CAddress> m_anchors; - - /** SipHasher seeds for deterministic randomness */ - const uint64_t nSeed0, nSeed1; - - /** flag for waking the message processor. */ - bool fMsgProcWake GUARDED_BY(mutexMsgProc); - - std::condition_variable condMsgProc; - Mutex mutexMsgProc; - std::atomic<bool> flagInterruptMsgProc{false}; - - CThreadInterrupt interruptNet; - - std::thread threadDNSAddressSeed; - std::thread threadSocketHandler; - std::thread threadOpenAddedConnections; - std::thread threadOpenConnections; - std::thread threadMessageHandler; - - /** flag for deciding to connect to an extra outbound peer, - * in excess of m_max_outbound_full_relay - * This takes the place of a feeler connection */ - std::atomic_bool m_try_another_outbound_peer; - - std::atomic<int64_t> m_next_send_inv_to_incoming{0}; - - /** - * A vector of -bind=<address>:<port>=onion arguments each of which is - * an address and port that are designated for incoming Tor connections. - */ - std::vector<CService> m_onion_binds; - - friend struct CConnmanTest; - friend struct ConnmanTestMsg; -}; +/** Convert ConnectionType enum to a string value */ +std::string ConnectionTypeAsString(ConnectionType conn_type); void Discover(); -void StartMapPort(); -void InterruptMapPort(); -void StopMapPort(); uint16_t GetListenPort(); -/** - * Interface for message handling - */ -class NetEventsInterface -{ -public: - virtual bool ProcessMessages(CNode* pnode, std::atomic<bool>& interrupt) = 0; - virtual bool SendMessages(CNode* pnode) = 0; - virtual void InitializeNode(CNode* pnode) = 0; - virtual void FinalizeNode(const CNode& node, bool& update_connection_time) = 0; - -protected: - /** - * Protected destructor so that instances can only be deleted by derived classes. - * If that restriction is no longer desired, this should be made public and virtual. - */ - ~NetEventsInterface() = default; -}; - enum { LOCAL_NONE, // unknown LOCAL_IF, // address a local interface listens on LOCAL_BIND, // address explicit bound to - LOCAL_UPNP, // address reported by UPnP + LOCAL_MAPPED, // address reported by UPnP or NAT-PMP LOCAL_MANUAL, // address explicitly specified (-externalip=) LOCAL_MAX @@ -669,7 +214,6 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) extern bool fDiscover; extern bool fListen; -extern bool g_relay_txes; /** Subversion as sent to the P2P network in `version` messages */ extern std::string strSubVersion; @@ -701,14 +245,14 @@ public: int nVersion; std::string cleanSubVer; bool fInbound; - bool m_manual_connection; - int nStartingHeight; + bool m_bip152_highbandwidth_to; + bool m_bip152_highbandwidth_from; + int m_starting_height; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; uint64_t nRecvBytes; mapMsgCmdSize mapRecvBytesPerMsgCmd; NetPermissionFlags m_permissionFlags; - bool m_legacyWhitelisted; int64_t m_ping_usec; int64_t m_ping_wait_usec; int64_t m_min_ping_usec; @@ -719,14 +263,13 @@ public: CAddress addr; // Bind address of our side of the connection CAddress addrBind; - // Name of the network the peer connected through - std::string m_network; + // Network the peer connected through + Network m_network; uint32_t m_mapped_as; - std::string m_conn_type_string; + ConnectionType m_conn_type; }; - /** Transport protocol agnostic message container. * Ideally it should only contain receive time, payload, * command and size. @@ -757,8 +300,8 @@ public: virtual bool Complete() const = 0; // set the serialization context version virtual void SetVersion(int version) = 0; - // read and deserialize data - virtual int Read(const char *data, unsigned int bytes) = 0; + /** read and deserialize data, advances msg_bytes data pointer */ + virtual int Read(Span<const uint8_t>& msg_bytes) = 0; // decomposes a message from the context virtual Optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err) = 0; virtual ~TransportDeserializer() {} @@ -779,8 +322,8 @@ private: unsigned int nDataPos; const uint256& GetMessageHash() const; - int readHeader(const char *pch, unsigned int nBytes); - int readData(const char *pch, unsigned int nBytes); + int readHeader(Span<const uint8_t> msg_bytes); + int readData(Span<const uint8_t> msg_bytes); void Reset() { vRecv.clear(); @@ -814,9 +357,14 @@ public: hdrbuf.SetVersion(nVersionIn); vRecv.SetVersion(nVersionIn); } - int Read(const char *pch, unsigned int nBytes) override { - int ret = in_data ? readData(pch, nBytes) : readHeader(pch, nBytes); - if (ret < 0) Reset(); + int Read(Span<const uint8_t>& msg_bytes) override + { + int ret = in_data ? readData(msg_bytes) : readHeader(msg_bytes); + if (ret < 0) { + Reset(); + } else { + msg_bytes = msg_bytes.subspan(ret); + } return ret; } Optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err_raw_size) override; @@ -846,16 +394,18 @@ public: std::unique_ptr<TransportDeserializer> m_deserializer; std::unique_ptr<TransportSerializer> m_serializer; - // socket + NetPermissionFlags m_permissionFlags{PF_NONE}; std::atomic<ServiceFlags> nServices{NODE_NONE}; SOCKET hSocket GUARDED_BY(cs_hSocket); - size_t nSendSize{0}; // total size of all vSendMsg entries - size_t nSendOffset{0}; // offset inside the first vSendMsg already sent + /** Total size of all vSendMsg entries */ + size_t nSendSize GUARDED_BY(cs_vSend){0}; + /** Offset inside the first vSendMsg already sent */ + size_t nSendOffset GUARDED_BY(cs_vSend){0}; uint64_t nSendBytes GUARDED_BY(cs_vSend){0}; std::deque<std::vector<unsigned char>> vSendMsg GUARDED_BY(cs_vSend); - RecursiveMutex cs_vSend; - RecursiveMutex cs_hSocket; - RecursiveMutex cs_vRecv; + Mutex cs_vSend; + Mutex cs_hSocket; + Mutex cs_vRecv; RecursiveMutex cs_vProcessMsg; std::list<CNetMessage> vProcessMsg GUARDED_BY(cs_vProcessMsg); @@ -884,8 +434,6 @@ public: bool HasPermission(NetPermissionFlags permission) const { return NetPermissions::HasFlag(m_permissionFlags, permission); } - // This boolean is unusued in actual processing, only present for backward compatibility at RPC/QT level - bool m_legacyWhitelisted{false}; bool fClient{false}; // set by version message bool m_limited_node{false}; //after BIP159, set by version message /** @@ -947,6 +495,9 @@ public: /* Whether we send addr messages over this connection */ bool RelayAddrsWithConn() const { + // Don't relay addr messages to peers that we connect to as block-relay-only + // peers (to prevent adversaries from inferring these links from addr + // traffic). return m_conn_type != ConnectionType::BLOCK_RELAY; } @@ -977,13 +528,10 @@ public: */ Network ConnectedThroughNetwork() const; -protected: - mapMsgCmdSize mapSendBytesPerMsgCmd; - mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); - -public: - uint256 hashContinue; - std::atomic<int> nStartingHeight{-1}; + // We selected peer as (compact blocks) high-bandwidth peer (BIP152) + std::atomic<bool> m_bip152_highbandwidth_to{false}; + // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) + std::atomic<bool> m_bip152_highbandwidth_from{false}; // flood relay std::vector<CAddress> vAddrToSend; @@ -992,12 +540,6 @@ public: std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; - // List of block ids we still have announce. - // There is no final sorting before sending, as they are always sent immediately - // and in the order requested. - std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - Mutex cs_inventory; - struct TxRelay { mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - @@ -1015,12 +557,11 @@ public: // Used for BIP35 mempool sending bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; // Last time a "MEMPOOL" request was serviced. - std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; + std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; std::chrono::microseconds nNextInvSend{0}; - RecursiveMutex cs_feeFilter; - // Minimum fee rate with which to filter inv's to this node - CAmount minFeeFilter GUARDED_BY(cs_feeFilter){0}; + /** Minimum fee rate with which to filter inv's to this node */ + std::atomic<CAmount> minFeeFilter{0}; CAmount lastSentFeeFilter{0}; int64_t nextSendTimeFeeFilter{0}; }; @@ -1028,9 +569,6 @@ public: // m_tx_relay == nullptr if we're not relaying transactions with this peer std::unique_ptr<TxRelay> m_tx_relay; - // Used for headers announcements - unfiltered blocks to relay - std::vector<uint256> vBlockHashesToAnnounce GUARDED_BY(cs_inventory); - /** UNIX epoch time of the last block received from this peer that we had * not yet seen (e.g. not already received from another peer), that passed * preliminary validity checks and was saved to disk, even if we don't @@ -1048,7 +586,7 @@ public: // The pong reply we're expecting, or 0 if no pong expected. std::atomic<uint64_t> nPingNonceSent{0}; /** When the last ping was sent, or 0 if no ping was ever sent */ - std::atomic<std::chrono::microseconds> m_ping_start{std::chrono::microseconds{0}}; + std::atomic<std::chrono::microseconds> m_ping_start{0us}; // Last measured round-trip time. std::atomic<int64_t> nPingUsecTime{0}; // Best measured round-trip time. @@ -1056,50 +594,11 @@ public: // Whether a ping is requested. std::atomic<bool> fPingQueued{false}; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn, ConnectionType conn_type_in, bool inbound_onion = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion = false); ~CNode(); CNode(const CNode&) = delete; CNode& operator=(const CNode&) = delete; -private: - const NodeId id; - const uint64_t nLocalHostNonce; - const ConnectionType m_conn_type; - std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; - - //! Services offered to this peer. - //! - //! This is supplied by the parent CConnman during peer connection - //! (CConnman::ConnectNode()) from its attribute of the same name. - //! - //! This is const because there is no protocol defined for renegotiating - //! services initially offered to a peer. The set of local services we - //! offer should not change after initialization. - //! - //! An interesting example of this is NODE_NETWORK and initial block - //! download: a node which starts up from scratch doesn't have any blocks - //! to serve, but still advertises NODE_NETWORK because it will eventually - //! fulfill this role after IBD completes. P2P code is written in such a - //! way that it can gracefully handle peers who don't make good on their - //! service advertisements. - const ServiceFlags nLocalServices; - - const int nMyStartingHeight; - NetPermissionFlags m_permissionFlags{ PF_NONE }; - std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread - - mutable RecursiveMutex cs_addrName; - std::string addrName GUARDED_BY(cs_addrName); - - // Our address, as reported by the peer - CService addrLocal GUARDED_BY(cs_addrLocal); - mutable RecursiveMutex cs_addrLocal; - - //! Whether this peer connected via our Tor onion service. - const bool m_inbound_onion{false}; - -public: - NodeId GetId() const { return id; } @@ -1108,20 +607,26 @@ public: return nLocalHostNonce; } - int GetMyStartingHeight() const { - return nMyStartingHeight; - } - int GetRefCount() const { assert(nRefCount >= 0); return nRefCount; } - bool ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete); + /** + * Receive bytes from the buffer and deserialize them into messages. + * + * @param[in] msg_bytes The raw data + * @param[out] complete Set True if at least one message has been + * deserialized and is ready to be processed + * @return True if the peer should stay connected, + * False if the peer should be disconnected from. + */ + bool ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete); void SetCommonVersion(int greatest_common_version) { + Assume(m_greatest_common_version == INIT_PROTO_VERSION); m_greatest_common_version = greatest_common_version; } int GetCommonVersion() const @@ -1144,26 +649,29 @@ public: nRefCount--; } - - void AddAddressKnown(const CAddress& _addr) { assert(m_addr_known); m_addr_known->insert(_addr.GetKey()); } - void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) + /** + * Whether the peer supports the address. For example, a peer that does not + * implement BIP155 cannot receive Tor v3 addresses because it requires + * ADDRv2 (BIP155) encoding. + */ + bool IsAddrCompatible(const CAddress& addr) const { - // Whether the peer supports the address in `_addr`. For example, - // nodes that do not implement BIP155 cannot receive Tor v3 addresses - // because they require ADDRv2 (BIP155) encoding. - const bool addr_format_supported = m_wants_addrv2 || _addr.IsAddrV1Compatible(); + return m_wants_addrv2 || addr.IsAddrV1Compatible(); + } + void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) + { // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. assert(m_addr_known); - if (_addr.IsValid() && !m_addr_known->contains(_addr.GetKey()) && addr_format_supported) { + if (_addr.IsValid() && !m_addr_known->contains(_addr.GetKey()) && IsAddrCompatible(_addr)) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { @@ -1172,7 +680,6 @@ public: } } - void AddKnownTx(const uint256& hash) { if (m_tx_relay != nullptr) { @@ -1203,7 +710,524 @@ public: //! Sets the addrName only if it was not previously set void MaybeSetAddrName(const std::string& addrNameIn); - std::string ConnectionTypeAsString() const; + std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } + + /** Whether this peer is an inbound onion, e.g. connected via our Tor onion service. */ + bool IsInboundOnion() const { return m_inbound_onion; } + +private: + const NodeId id; + const uint64_t nLocalHostNonce; + const ConnectionType m_conn_type; + std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; + + //! Services offered to this peer. + //! + //! This is supplied by the parent CConnman during peer connection + //! (CConnman::ConnectNode()) from its attribute of the same name. + //! + //! This is const because there is no protocol defined for renegotiating + //! services initially offered to a peer. The set of local services we + //! offer should not change after initialization. + //! + //! An interesting example of this is NODE_NETWORK and initial block + //! download: a node which starts up from scratch doesn't have any blocks + //! to serve, but still advertises NODE_NETWORK because it will eventually + //! fulfill this role after IBD completes. P2P code is written in such a + //! way that it can gracefully handle peers who don't make good on their + //! service advertisements. + const ServiceFlags nLocalServices; + + std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread + + mutable RecursiveMutex cs_addrName; + std::string addrName GUARDED_BY(cs_addrName); + + // Our address, as reported by the peer + CService addrLocal GUARDED_BY(cs_addrLocal); + mutable RecursiveMutex cs_addrLocal; + + //! Whether this peer is an inbound onion, e.g. connected via our Tor onion service. + const bool m_inbound_onion{false}; + + mapMsgCmdSize mapSendBytesPerMsgCmd GUARDED_BY(cs_vSend); + mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); +}; + +/** + * Interface for message handling + */ +class NetEventsInterface +{ +public: + /** Initialize a peer (setup state, queue any initial messages) */ + virtual void InitializeNode(CNode* pnode) = 0; + + /** Handle removal of a peer (clear state) */ + virtual void FinalizeNode(const CNode& node, bool& update_connection_time) = 0; + + /** + * Process protocol messages received from a given node + * + * @param[in] pnode The node which we have received messages from. + * @param[in] interrupt Interrupt condition for processing threads + * @return True if there is more work to be done + */ + virtual bool ProcessMessages(CNode* pnode, std::atomic<bool>& interrupt) = 0; + + /** + * Send queued protocol messages to a given node. + * + * @param[in] pnode The node which we are sending messages to. + * @return True if there is more work to be done + */ + virtual bool SendMessages(CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_sendProcessing) = 0; + + +protected: + /** + * Protected destructor so that instances can only be deleted by derived classes. + * If that restriction is no longer desired, this should be made public and virtual. + */ + ~NetEventsInterface() = default; +}; + +class CConnman +{ +public: + + enum NumConnections { + CONNECTIONS_NONE = 0, + CONNECTIONS_IN = (1U << 0), + CONNECTIONS_OUT = (1U << 1), + CONNECTIONS_ALL = (CONNECTIONS_IN | CONNECTIONS_OUT), + }; + + struct Options + { + ServiceFlags nLocalServices = NODE_NONE; + int nMaxConnections = 0; + int m_max_outbound_full_relay = 0; + int m_max_outbound_block_relay = 0; + int nMaxAddnode = 0; + int nMaxFeeler = 0; + CClientUIInterface* uiInterface = nullptr; + NetEventsInterface* m_msgproc = nullptr; + BanMan* m_banman = nullptr; + unsigned int nSendBufferMaxSize = 0; + unsigned int nReceiveFloodSize = 0; + uint64_t nMaxOutboundLimit = 0; + int64_t m_peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT; + std::vector<std::string> vSeedNodes; + std::vector<NetWhitelistPermissions> vWhitelistedRange; + std::vector<NetWhitebindPermissions> vWhiteBinds; + std::vector<CService> vBinds; + std::vector<CService> onion_binds; + bool m_use_addrman_outgoing = true; + std::vector<std::string> m_specified_outgoing; + std::vector<std::string> m_added_nodes; + std::vector<bool> m_asmap; + }; + + void Init(const Options& connOptions) { + nLocalServices = connOptions.nLocalServices; + nMaxConnections = connOptions.nMaxConnections; + m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); + m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; + m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; + nMaxAddnode = connOptions.nMaxAddnode; + nMaxFeeler = connOptions.nMaxFeeler; + m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; + clientInterface = connOptions.uiInterface; + m_banman = connOptions.m_banman; + m_msgproc = connOptions.m_msgproc; + nSendBufferMaxSize = connOptions.nSendBufferMaxSize; + nReceiveFloodSize = connOptions.nReceiveFloodSize; + m_peer_connect_timeout = connOptions.m_peer_connect_timeout; + { + LOCK(cs_totalBytesSent); + nMaxOutboundLimit = connOptions.nMaxOutboundLimit; + } + vWhitelistedRange = connOptions.vWhitelistedRange; + { + LOCK(cs_vAddedNodes); + vAddedNodes = connOptions.m_added_nodes; + } + m_onion_binds = connOptions.onion_binds; + } + + CConnman(uint64_t seed0, uint64_t seed1, bool network_active = true); + ~CConnman(); + bool Start(CScheduler& scheduler, const Options& options); + + void StopThreads(); + void StopNodes(); + void Stop() + { + StopThreads(); + StopNodes(); + }; + + void Interrupt(); + bool GetNetworkActive() const { return fNetworkActive; }; + bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; + void SetNetworkActive(bool active); + void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant* grantOutbound, const char* strDest, ConnectionType conn_type); + bool CheckIncomingNonce(uint64_t nonce); + + bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); + + void PushMessage(CNode* pnode, CSerializedNetMsg&& msg); + + using NodeFn = std::function<void(CNode*)>; + void ForEachNode(const NodeFn& func) + { + LOCK(cs_vNodes); + for (auto&& node : vNodes) { + if (NodeFullyConnected(node)) + func(node); + } + }; + + void ForEachNode(const NodeFn& func) const + { + LOCK(cs_vNodes); + for (auto&& node : vNodes) { + if (NodeFullyConnected(node)) + func(node); + } + }; + + template<typename Callable, typename CallableAfter> + void ForEachNodeThen(Callable&& pre, CallableAfter&& post) + { + LOCK(cs_vNodes); + for (auto&& node : vNodes) { + if (NodeFullyConnected(node)) + pre(node); + } + post(); + }; + + template<typename Callable, typename CallableAfter> + void ForEachNodeThen(Callable&& pre, CallableAfter&& post) const + { + LOCK(cs_vNodes); + for (auto&& node : vNodes) { + if (NodeFullyConnected(node)) + pre(node); + } + post(); + }; + + // Addrman functions + void SetServices(const CService &addr, ServiceFlags nServices); + void MarkAddressGood(const CAddress& addr); + bool AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); + std::vector<CAddress> GetAddresses(size_t max_addresses, size_t max_pct); + /** + * Cache is used to minimize topology leaks, so it should + * be used for all non-trusted calls, for example, p2p. + * A non-malicious call (from RPC or a peer with addr permission) should + * call the function without a parameter to avoid using the cache. + */ + std::vector<CAddress> GetAddresses(CNode& requestor, size_t max_addresses, size_t max_pct); + + // This allows temporarily exceeding m_max_outbound_full_relay, with the goal of finding + // a peer that is better than all our current peers. + void SetTryNewOutboundPeer(bool flag); + bool GetTryNewOutboundPeer(); + + void StartExtraBlockRelayPeers() { + LogPrint(BCLog::NET, "net: enabling extra block-relay-only peers\n"); + m_start_extra_block_relay_peers = true; + } + + // Return the number of outbound peers we have in excess of our target (eg, + // if we previously called SetTryNewOutboundPeer(true), and have since set + // to false, we may have extra peers that we wish to disconnect). This may + // return a value less than (num_outbound_connections - num_outbound_slots) + // in cases where some outbound connections are not yet fully connected, or + // not yet fully disconnected. + int GetExtraFullOutboundCount(); + // Count the number of block-relay-only peers we have over our limit. + int GetExtraBlockRelayCount(); + + bool AddNode(const std::string& node); + bool RemoveAddedNode(const std::string& node); + std::vector<AddedNodeInfo> GetAddedNodeInfo(); + + /** + * Attempts to open a connection. Currently only used from tests. + * + * @param[in] address Address of node to try connecting to + * @param[in] conn_type ConnectionType::OUTBOUND or ConnectionType::BLOCK_RELAY + * @return bool Returns false if there are no available + * slots for this connection: + * - conn_type not a supported ConnectionType + * - Max total outbound connection capacity filled + * - Max connection capacity for type is filled + */ + bool AddConnection(const std::string& address, ConnectionType conn_type); + + size_t GetNodeCount(NumConnections num); + void GetNodeStats(std::vector<CNodeStats>& vstats); + bool DisconnectNode(const std::string& node); + bool DisconnectNode(const CSubNet& subnet); + bool DisconnectNode(const CNetAddr& addr); + bool DisconnectNode(NodeId id); + + //! Used to convey which local services we are offering peers during node + //! connection. + //! + //! The data returned by this is used in CNode construction, + //! which is used to advertise which services we are offering + //! that peer during `net_processing.cpp:PushNodeVersion()`. + ServiceFlags GetLocalServices() const; + + uint64_t GetMaxOutboundTarget(); + std::chrono::seconds GetMaxOutboundTimeframe(); + + //! check if the outbound target is reached + //! if param historicalBlockServingLimit is set true, the function will + //! response true if the limit for serving historical blocks has been reached + bool OutboundTargetReached(bool historicalBlockServingLimit); + + //! response the bytes left in the current max outbound cycle + //! in case of no limit, it will always response 0 + uint64_t GetOutboundTargetBytesLeft(); + + //! returns the time left in the current max outbound cycle + //! in case of no limit, it will always return 0 + std::chrono::seconds GetMaxOutboundTimeLeftInCycle(); + + uint64_t GetTotalBytesRecv(); + uint64_t GetTotalBytesSent(); + + /** Get a unique deterministic randomizer. */ + CSipHasher GetDeterministicRandomizer(uint64_t id) const; + + unsigned int GetReceiveFloodSize() const; + + void WakeMessageHandler(); + + /** Attempts to obfuscate tx time through exponentially distributed emitting. + Works assuming that a single interval is used. + Variable intervals will result in privacy decrease. + */ + int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + + void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } + +private: + struct ListenSocket { + public: + SOCKET socket; + inline void AddSocketPermissionFlags(NetPermissionFlags& flags) const { NetPermissions::AddFlag(flags, m_permissions); } + ListenSocket(SOCKET socket_, NetPermissionFlags permissions_) : socket(socket_), m_permissions(permissions_) {} + private: + NetPermissionFlags m_permissions; + }; + + bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); + bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); + bool InitBinds( + const std::vector<CService>& binds, + const std::vector<NetWhitebindPermissions>& whiteBinds, + const std::vector<CService>& onion_binds); + + void ThreadOpenAddedConnections(); + void AddAddrFetch(const std::string& strDest); + void ProcessAddrFetch(); + void ThreadOpenConnections(std::vector<std::string> connect); + void ThreadMessageHandler(); + void AcceptConnection(const ListenSocket& hListenSocket); + void DisconnectNodes(); + void NotifyNumConnectionsChanged(); + void InactivityCheck(CNode *pnode) const; + bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); + void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); + void SocketHandler(); + void ThreadSocketHandler(); + void ThreadDNSAddressSeed(); + + uint64_t CalculateKeyedNetGroup(const CAddress& ad) const; + + CNode* FindNode(const CNetAddr& ip); + CNode* FindNode(const CSubNet& subNet); + CNode* FindNode(const std::string& addrName); + CNode* FindNode(const CService& addr); + + /** + * Determine whether we're already connected to a given address, in order to + * avoid initiating duplicate connections. + */ + bool AlreadyConnectedToAddress(const CAddress& addr); + + bool AttemptToEvictConnection(); + CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type); + void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const; + + void DeleteNode(CNode* pnode); + + NodeId GetNewNodeId(); + + size_t SocketSendData(CNode& node) const EXCLUSIVE_LOCKS_REQUIRED(node.cs_vSend); + void DumpAddresses(); + + // Network stats + void RecordBytesRecv(uint64_t bytes); + void RecordBytesSent(uint64_t bytes); + + /** + * Return vector of current BLOCK_RELAY peers. + */ + std::vector<CAddress> GetCurrentBlockRelayOnlyConns() const; + + // Whether the node should be passed out in ForEach* callbacks + static bool NodeFullyConnected(const CNode* pnode); + + // Network usage totals + RecursiveMutex cs_totalBytesRecv; + RecursiveMutex cs_totalBytesSent; + uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; + uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; + + // outbound limit & stats + uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent) {0}; + std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent) {0}; + uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); + + // P2P timeout in seconds + int64_t m_peer_connect_timeout; + + // Whitelisted ranges. Any node connecting from these is automatically + // whitelisted (as well as those connecting to whitelisted binds). + std::vector<NetWhitelistPermissions> vWhitelistedRange; + + unsigned int nSendBufferMaxSize{0}; + unsigned int nReceiveFloodSize{0}; + + std::vector<ListenSocket> vhListenSocket; + std::atomic<bool> fNetworkActive{true}; + bool fAddressesInitialized{false}; + CAddrMan addrman; + std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); + RecursiveMutex m_addr_fetches_mutex; + std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); + RecursiveMutex cs_vAddedNodes; + std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); + std::list<CNode*> vNodesDisconnected; + mutable RecursiveMutex cs_vNodes; + std::atomic<NodeId> nLastNodeId{0}; + unsigned int nPrevNodeCount{0}; + + /** + * Cache responses to addr requests to minimize privacy leak. + * Attack example: scraping addrs in real-time may allow an attacker + * to infer new connections of the victim by detecting new records + * with fresh timestamps (per self-announcement). + */ + struct CachedAddrResponse { + std::vector<CAddress> m_addrs_response_cache; + std::chrono::microseconds m_cache_entry_expiration{0}; + }; + + /** + * Addr responses stored in different caches + * per (network, local socket) prevent cross-network node identification. + * If a node for example is multi-homed under Tor and IPv6, + * a single cache (or no cache at all) would let an attacker + * to easily detect that it is the same node by comparing responses. + * Indexing by local socket prevents leakage when a node has multiple + * listening addresses on the same network. + * + * The used memory equals to 1000 CAddress records (or around 40 bytes) per + * distinct Network (up to 5) we have/had an inbound peer from, + * resulting in at most ~196 KB. Every separate local socket may + * add up to ~196 KB extra. + */ + std::map<uint64_t, CachedAddrResponse> m_addr_response_caches; + + /** + * Services this instance offers. + * + * This data is replicated in each CNode instance we create during peer + * connection (in ConnectNode()) under a member also called + * nLocalServices. + * + * This data is not marked const, but after being set it should not + * change. See the note in CNode::nLocalServices documentation. + * + * \sa CNode::nLocalServices + */ + ServiceFlags nLocalServices; + + std::unique_ptr<CSemaphore> semOutbound; + std::unique_ptr<CSemaphore> semAddnode; + int nMaxConnections; + + // How many full-relay (tx, block, addr) outbound peers we want + int m_max_outbound_full_relay; + + // How many block-relay only outbound peers we want + // We do not relay tx or addr messages with these peers + int m_max_outbound_block_relay; + + int nMaxAddnode; + int nMaxFeeler; + int m_max_outbound; + bool m_use_addrman_outgoing; + CClientUIInterface* clientInterface; + NetEventsInterface* m_msgproc; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ + BanMan* m_banman; + + /** + * Addresses that were saved during the previous clean shutdown. We'll + * attempt to make block-relay-only connections to them. + */ + std::vector<CAddress> m_anchors; + + /** SipHasher seeds for deterministic randomness */ + const uint64_t nSeed0, nSeed1; + + /** flag for waking the message processor. */ + bool fMsgProcWake GUARDED_BY(mutexMsgProc); + + std::condition_variable condMsgProc; + Mutex mutexMsgProc; + std::atomic<bool> flagInterruptMsgProc{false}; + + CThreadInterrupt interruptNet; + + std::thread threadDNSAddressSeed; + std::thread threadSocketHandler; + std::thread threadOpenAddedConnections; + std::thread threadOpenConnections; + std::thread threadMessageHandler; + + /** flag for deciding to connect to an extra outbound peer, + * in excess of m_max_outbound_full_relay + * This takes the place of a feeler connection */ + std::atomic_bool m_try_another_outbound_peer; + + /** flag for initiating extra block-relay-only peer connections. + * this should only be enabled after initial chain sync has occurred, + * as these connections are intended to be short-lived and low-bandwidth. + */ + std::atomic_bool m_start_extra_block_relay_peers{false}; + + std::atomic<int64_t> m_next_send_inv_to_incoming{0}; + + /** + * A vector of -bind=<address>:<port>=onion arguments each of which is + * an address and port that are designated for incoming Tor connections. + */ + std::vector<CService> m_onion_binds; + + friend struct CConnmanTest; + friend struct ConnmanTestMsg; }; /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ @@ -1215,4 +1239,21 @@ inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; } +struct NodeEvictionCandidate +{ + NodeId id; + int64_t nTimeConnected; + int64_t nMinPingUsecTime; + int64_t nLastBlockTime; + int64_t nLastTXTime; + bool fRelevantServices; + bool fRelayTxes; + bool fBloomFilter; + uint64_t nKeyedNetGroup; + bool prefer_evict; + bool m_is_local; +}; + +[[nodiscard]] Optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates); + #endif // BITCOIN_NET_H diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index d40fdfb113..1fdcd97593 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c649cf7757..cf73b1dae2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -26,6 +26,7 @@ #include <streams.h> #include <tinyformat.h> #include <txmempool.h> +#include <txrequest.h> #include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> #include <util/system.h> @@ -168,6 +169,186 @@ void EraseOrphansFor(NodeId peer); // Internal stuff namespace { +/** + * Data structure for an individual peer. This struct is not protected by + * cs_main since it does not contain validation-critical data. + * + * Memory is owned by shared pointers and this object is destructed when + * the refcount drops to zero. + * + * Mutexes inside this struct must not be held when locking m_peer_mutex. + * + * TODO: move most members from CNodeState to this structure. + * TODO: move remaining application-layer data members from CNode to this structure. + */ +struct Peer { + /** Same id as the CNode object for this peer */ + const NodeId m_id{0}; + + /** Protects misbehavior data members */ + Mutex m_misbehavior_mutex; + /** Accumulated misbehavior score for this peer */ + int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; + /** Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). */ + bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; + + /** Protects block inventory data members */ + Mutex m_block_inv_mutex; + /** List of blocks that we'll announce via an `inv` message. + * There is no final sorting before sending, as they are always sent + * immediately and in the order requested. */ + std::vector<uint256> m_blocks_for_inv_relay GUARDED_BY(m_block_inv_mutex); + /** Unfiltered list of blocks that we'd like to announce via a `headers` + * message. If we can't announce via a `headers` message, we'll fall back to + * announcing via `inv`. */ + std::vector<uint256> m_blocks_for_headers_relay GUARDED_BY(m_block_inv_mutex); + /** The final block hash that we sent in an `inv` message to this peer. + * When the peer requests this block, we send an `inv` message to trigger + * the peer to request the next sequence of block hashes. + * Most peers use headers-first syncing, which doesn't use this mechanism */ + uint256 m_continuation_block GUARDED_BY(m_block_inv_mutex) {}; + + /** This peer's reported block height when we connected */ + std::atomic<int> m_starting_height{-1}; + + /** Set of txids to reconsider once their parent transactions have been accepted **/ + std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); + + /** Protects m_getdata_requests **/ + Mutex m_getdata_requests_mutex; + /** Work queue of items requested by this peer **/ + std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); + + explicit Peer(NodeId id) : m_id(id) {} +}; + +using PeerRef = std::shared_ptr<Peer>; + +class PeerManagerImpl final : public PeerManager +{ +public: + PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs); + + /** Overridden from CValidationInterface. */ + void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; + void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; + void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; + void BlockChecked(const CBlock& block, const BlockValidationState& state) override; + void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; + + /** Implement NetEventsInterface */ + void InitializeNode(CNode* pnode) override; + void FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) override; + bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override; + bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); + + /** Implement PeerManager */ + void CheckForStaleTipAndEvictPeers() override; + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) override; + bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } + void SetBestHeight(int height) override { m_best_height = height; }; + void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) override; + void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override; + +private: + /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ + void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ + void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ + void ReattemptInitialBroadcast(CScheduler& scheduler) const; + + /** Get a shared pointer to the Peer object. + * May return an empty shared_ptr if the Peer object can't be found. */ + PeerRef GetPeerRef(NodeId id) const; + + /** Get a shared pointer to the Peer object and remove it from m_peer_map. + * May return an empty shared_ptr if the Peer object can't be found. */ + PeerRef RemovePeer(NodeId id); + + /** + * Potentially mark a node discouraged based on the contents of a BlockValidationState object + * + * @param[in] via_compact_block this bool is passed in because net_processing should + * punish peers differently depending on whether the data was provided in a compact + * block message or not. If the compact block had a valid header, but contained invalid + * txs, the peer should not be punished. See BIP 152. + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message = ""); + + /** + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = ""); + + /** Maybe disconnect a peer and discourage future connections from its address. + * + * @param[in] pnode The node to check. + * @return True if the peer was marked for disconnection in this function + */ + bool MaybeDiscourageAndDisconnect(CNode& pnode); + + void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); + /** Process a single headers message from a peer. */ + void ProcessHeadersMessage(CNode& pfrom, const Peer& peer, + const std::vector<CBlockHeader>& headers, + bool via_compact_block); + + void SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req); + + /** Register with TxRequestTracker that an INV has been received from a + * peer. The announcement parameters are decided in PeerManager and then + * passed to TxRequestTracker. */ + void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + /** Send a version message to a peer */ + void PushNodeVersion(CNode& pnode, int64_t nTime); + + const CChainParams& m_chainparams; + CConnman& m_connman; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ + BanMan* const m_banman; + ChainstateManager& m_chainman; + CTxMemPool& m_mempool; + TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + + /** The height of the best chain */ + std::atomic<int> m_best_height{-1}; + + int64_t m_stale_tip_check_time; //!< Next time to check for stale tip + + /** Whether this node is running in blocks only mode */ + const bool m_ignore_incoming_txs; + + /** Whether we've completed initial sync yet, for determining when to turn + * on extra block-relay-only peers. */ + bool m_initial_sync_finished{false}; + + /** Protects m_peer_map. This mutex must not be locked while holding a lock + * on any of the mutexes inside a Peer object. */ + mutable Mutex m_peer_mutex; + /** + * Map of all Peer objects, keyed by peer id. This map is protected + * by the m_peer_mutex. Once a shared pointer reference is + * taken, the lock may be released. Individual fields are protected by + * their own locks. + */ + std::map<NodeId, PeerRef> m_peer_map GUARDED_BY(m_peer_mutex); +}; +} // namespace + +namespace { /** Number of nodes with fSyncStarted. */ int nSyncStarted GUARDED_BY(cs_main) = 0; @@ -290,8 +471,6 @@ namespace { struct CNodeState { //! The peer's address const CService address; - //! Whether we have a fully established connection. - bool fCurrentlyConnected; //! The best known block we know this peer has announced. const CBlockIndex *pindexBestKnownBlock; //! The hash of the last unknown block this peer has announced. @@ -378,19 +557,15 @@ struct CNodeState { //! Whether this peer is an inbound connection bool m_is_inbound; - //! Whether this peer is a manual connection - bool m_is_manual_connection; - //! A rolling bloom filter of all announced tx CInvs to this peer. CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; //! Whether this peer relays txs via wtxid bool m_wtxid_relay{false}; - CNodeState(CAddress addrIn, bool is_inbound, bool is_manual) - : address(addrIn), m_is_inbound(is_inbound), m_is_manual_connection(is_manual) + CNodeState(CAddress addrIn, bool is_inbound) + : address(addrIn), m_is_inbound(is_inbound) { - fCurrentlyConnected = false; pindexBestKnownBlock = nullptr; hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = nullptr; @@ -425,58 +600,6 @@ static CNodeState *State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return &it->second; } -/** - * Data structure for an individual peer. This struct is not protected by - * cs_main since it does not contain validation-critical data. - * - * Memory is owned by shared pointers and this object is destructed when - * the refcount drops to zero. - * - * TODO: move most members from CNodeState to this structure. - * TODO: move remaining application-layer data members from CNode to this structure. - */ -struct Peer { - /** Same id as the CNode object for this peer */ - const NodeId m_id{0}; - - /** Protects misbehavior data members */ - Mutex m_misbehavior_mutex; - /** Accumulated misbehavior score for this peer */ - int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; - /** Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). */ - bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; - - /** Set of txids to reconsider once their parent transactions have been accepted **/ - std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); - - /** Protects m_getdata_requests **/ - Mutex m_getdata_requests_mutex; - /** Work queue of items requested by this peer **/ - std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - - Peer(NodeId id) : m_id(id) {} -}; - -using PeerRef = std::shared_ptr<Peer>; - -/** - * Map of all Peer objects, keyed by peer id. This map is protected - * by the global g_peer_mutex. Once a shared pointer reference is - * taken, the lock may be released. Individual fields are protected by - * their own locks. - */ -Mutex g_peer_mutex; -static std::map<NodeId, PeerRef> g_peer_map GUARDED_BY(g_peer_mutex); - -/** Get a shared pointer to the Peer object. - * May return nullptr if the Peer object can't be found. */ -static PeerRef GetPeerRef(NodeId id) -{ - LOCK(g_peer_mutex); - auto it = g_peer_map.find(id); - return it != g_peer_map.end() ? it->second : nullptr; -} - static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { nPreferredDownload -= state->fPreferredDownload; @@ -487,32 +610,6 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } -static void PushNodeVersion(CNode& pnode, CConnman& connman, int64_t nTime) -{ - // Note that pnode->GetLocalServices() is a reflection of the local - // services we were offering when the CNode object was created for this - // peer. - ServiceFlags nLocalNodeServices = pnode.GetLocalServices(); - uint64_t nonce = pnode.GetLocalNonce(); - int nNodeStartingHeight = pnode.GetMyStartingHeight(); - NodeId nodeid = pnode.GetId(); - CAddress addr = pnode.addr; - - CAddress addrYou = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? - addr : - CAddress(CService(), addr.nServices); - CAddress addrMe = CAddress(CService(), nLocalNodeServices); - - connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, ::g_relay_txes && pnode.m_tx_relay != nullptr)); - - if (fLogIPs) { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); - } else { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); - } -} - // Returns a bool indicating whether we requested this block. // Also used if a block was /not/ received and timed out or started with another peer static bool MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { @@ -638,11 +735,15 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connma // blocks using compact encodings. connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); + // save BIP152 bandwidth state: we select peer to be low-bandwidth + pnodeStop->m_bip152_highbandwidth_to = false; return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); + // save BIP152 bandwidth state: we select peer to be high-bandwidth + pfrom->m_bip152_highbandwidth_to = true; lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); @@ -763,7 +864,33 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec } // namespace -void PeerManager::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) +void PeerManagerImpl::PushNodeVersion(CNode& pnode, int64_t nTime) +{ + // Note that pnode->GetLocalServices() is a reflection of the local + // services we were offering when the CNode object was created for this + // peer. + ServiceFlags nLocalNodeServices = pnode.GetLocalServices(); + uint64_t nonce = pnode.GetLocalNonce(); + const int nNodeStartingHeight{m_best_height}; + NodeId nodeid = pnode.GetId(); + CAddress addr = pnode.addr; + + CAddress addrYou = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? + addr : + CAddress(CService(), addr.nServices); + CAddress addrMe = CAddress(CService(), nLocalNodeServices); + + m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + nonce, strSubVersion, nNodeStartingHeight, !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr)); + + if (fLogIPs) { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + } else { + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + } +} + +void PeerManagerImpl::AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) { AssertLockHeld(::cs_main); // For m_txrequest NodeId nodeid = node.GetId(); @@ -799,26 +926,27 @@ void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) if (state) state->m_last_block_announcement = time_in_seconds; } -void PeerManager::InitializeNode(CNode *pnode) { +void PeerManagerImpl::InitializeNode(CNode *pnode) +{ CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); NodeId nodeid = pnode->GetId(); { LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, pnode->IsInboundConn(), pnode->IsManualConn())); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, pnode->IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } { PeerRef peer = std::make_shared<Peer>(nodeid); - LOCK(g_peer_mutex); - g_peer_map.emplace_hint(g_peer_map.end(), nodeid, std::move(peer)); + LOCK(m_peer_mutex); + m_peer_map.emplace_hint(m_peer_map.end(), nodeid, std::move(peer)); } if (!pnode->IsInboundConn()) { - PushNodeVersion(*pnode, m_connman, GetTime()); + PushNodeVersion(*pnode, GetTime()); } } -void PeerManager::ReattemptInitialBroadcast(CScheduler& scheduler) const +void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) const { std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); @@ -839,17 +967,21 @@ void PeerManager::ReattemptInitialBroadcast(CScheduler& scheduler) const scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } -void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { +void PeerManagerImpl::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) +{ NodeId nodeid = node.GetId(); fUpdateConnectionTime = false; LOCK(cs_main); int misbehavior{0}; { - PeerRef peer = GetPeerRef(nodeid); + // We remove the PeerRef from g_peer_map here, but we don't always + // destruct the Peer. Sometimes another thread is still holding a + // PeerRef, so the refcount is >= 1. Be careful not to do any + // processing here that assumes Peer won't be changed before it's + // destructed. + PeerRef peer = RemovePeer(nodeid); assert(peer != nullptr); misbehavior = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); - LOCK(g_peer_mutex); - g_peer_map.erase(nodeid); } CNodeState *state = State(nodeid); assert(state != nullptr); @@ -857,8 +989,9 @@ void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { if (state->fSyncStarted) nSyncStarted--; - if (misbehavior == 0 && state->fCurrentlyConnected && !node.IsBlockOnlyConn()) { - // Note: we avoid changing visible addrman state for block-relay-only peers + if (node.fSuccessfullyConnected && misbehavior == 0 && + !node.IsBlockOnlyConn() && !node.IsInboundConn()) { + // Only change visible addrman state for outbound, full-relay peers fUpdateConnectionTime = true; } @@ -889,7 +1022,27 @@ void PeerManager::FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) { LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } -bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { +PeerRef PeerManagerImpl::GetPeerRef(NodeId id) const +{ + LOCK(m_peer_mutex); + auto it = m_peer_map.find(id); + return it != m_peer_map.end() ? it->second : nullptr; +} + +PeerRef PeerManagerImpl::RemovePeer(NodeId id) +{ + PeerRef ret; + LOCK(m_peer_mutex); + auto it = m_peer_map.find(id); + if (it != m_peer_map.end()) { + ret = std::move(it->second); + m_peer_map.erase(it); + } + return ret; +} + +bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) +{ { LOCK(cs_main); CNodeState* state = State(nodeid); @@ -905,7 +1058,7 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; - stats.m_misbehavior_score = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); + stats.m_starting_height = peer->m_starting_height; return true; } @@ -1046,7 +1199,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) return nEvicted; } -void PeerManager::Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) +void PeerManagerImpl::Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) { assert(howmuch > 0); @@ -1064,8 +1217,8 @@ void PeerManager::Misbehaving(const NodeId pnode, const int howmuch, const std:: } } -bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, - bool via_compact_block, const std::string& message) +bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message) { switch (state.GetResult()) { case BlockValidationResult::BLOCK_RESULT_UNSET: @@ -1087,8 +1240,8 @@ bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationSt } // Discourage outbound (but not inbound) peers if on an invalid chain. - // Exempt HB compact block peers and manual connections. - if (!via_compact_block && !node_state->m_is_inbound && !node_state->m_is_manual_connection) { + // Exempt HB compact block peers. Manual connections are always protected from discouragement. + if (!via_compact_block && !node_state->m_is_inbound) { Misbehaving(nodeid, 100, message); return true; } @@ -1114,7 +1267,7 @@ bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationSt return false; } -bool PeerManager::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) +bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) { switch (state.GetResult()) { case TxValidationResult::TX_RESULT_UNSET: @@ -1160,14 +1313,23 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, - CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) +std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs) +{ + return std::make_unique<PeerManagerImpl>(chainparams, connman, banman, scheduler, chainman, pool, ignore_incoming_txs); +} + +PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs) : m_chainparams(chainparams), m_connman(connman), m_banman(banman), m_chainman(chainman), m_mempool(pool), - m_stale_tip_check_time(0) + m_stale_tip_check_time(0), + m_ignore_incoming_txs(ignore_incoming_txs) { // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); @@ -1200,7 +1362,7 @@ PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, Ban * block, remember the recently confirmed transactions, and delete tracked * announcements for them. Also save the time of the last tip update. */ -void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { { LOCK(g_cs_orphans); @@ -1251,7 +1413,7 @@ void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, co } } -void PeerManager::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) +void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) { // To avoid relay problems with transactions that were previously // confirmed, clear our filter of recently confirmed transactions whenever @@ -1276,7 +1438,8 @@ static bool fWitnessesPresentInMostRecentCompactBlock GUARDED_BY(cs_most_recent_ * Maintain state about the best-seen block and fast-announce a compact block * to compatible peers. */ -void PeerManager::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { +void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) +{ std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); const CNetMsgMaker msgMaker(PROTOCOL_VERSION); @@ -1323,42 +1486,47 @@ void PeerManager::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ * Update our best height and announce any block hashes which weren't previously * in ::ChainActive() to our peers. */ -void PeerManager::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - const int nNewHeight = pindexNew->nHeight; - m_connman.SetBestHeight(nNewHeight); - +void PeerManagerImpl::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) +{ + SetBestHeight(pindexNew->nHeight); SetServiceFlagsIBDCache(!fInitialDownload); - if (!fInitialDownload) { - // Find the hashes of all blocks that weren't previously in the best chain. - std::vector<uint256> vHashes; - const CBlockIndex *pindexToAnnounce = pindexNew; - while (pindexToAnnounce != pindexFork) { - vHashes.push_back(pindexToAnnounce->GetBlockHash()); - pindexToAnnounce = pindexToAnnounce->pprev; - if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) { - // Limit announcements in case of a huge reorganization. - // Rely on the peer's synchronization mechanism in that case. - break; - } + + // Don't relay inventory during initial block download. + if (fInitialDownload) return; + + // Find the hashes of all blocks that weren't previously in the best chain. + std::vector<uint256> vHashes; + const CBlockIndex *pindexToAnnounce = pindexNew; + while (pindexToAnnounce != pindexFork) { + vHashes.push_back(pindexToAnnounce->GetBlockHash()); + pindexToAnnounce = pindexToAnnounce->pprev; + if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) { + // Limit announcements in case of a huge reorganization. + // Rely on the peer's synchronization mechanism in that case. + break; } - // Relay inventory, but don't relay old inventory during initial block download. - m_connman.ForEachNode([nNewHeight, &vHashes](CNode* pnode) { - LOCK(pnode->cs_inventory); - if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { - for (const uint256& hash : reverse_iterate(vHashes)) { - pnode->vBlockHashesToAnnounce.push_back(hash); - } + } + + { + LOCK(m_peer_mutex); + for (auto& it : m_peer_map) { + Peer& peer = *it.second; + LOCK(peer.m_block_inv_mutex); + for (const uint256& hash : reverse_iterate(vHashes)) { + peer.m_blocks_for_headers_relay.push_back(hash); } - }); - m_connman.WakeMessageHandler(); + } } + + m_connman.WakeMessageHandler(); } /** * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ -void PeerManager::BlockChecked(const CBlock& block, const BlockValidationState& state) { +void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationState& state) +{ LOCK(cs_main); const uint256 hash(block.GetHash()); @@ -1445,7 +1613,23 @@ void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& }); } -static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman) +/** + * Relay (gossip) an address to a few randomly chosen nodes. + * We choose the same nodes within a given 24h window (if the list of connected + * nodes does not change) and we don't relay to nodes that already know an + * address. So within 24h we will likely relay a given address once. This is to + * prevent a peer from unjustly giving their address better propagation by sending + * it to us repeatedly. + * @param[in] originator The peer that sent us the address. We don't want to relay it back. + * @param[in] addr Address to relay. + * @param[in] fReachable Whether the address' network is reachable. We relay unreachable + * addresses less. + * @param[in] connman Connection manager to choose nodes to relay to. + */ +static void RelayAddress(const CNode& originator, + const CAddress& addr, + bool fReachable, + const CConnman& connman) { if (!fReachable && !addr.IsRelayable()) return; @@ -1462,8 +1646,8 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; assert(nRelayNodes <= best.size()); - auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { - if (pnode->RelayAddrsWithConn()) { + auto sortfunc = [&best, &hasher, nRelayNodes, &originator, &addr](CNode* pnode) { + if (pnode->RelayAddrsWithConn() && pnode != &originator && pnode->IsAddrCompatible(addr)) { uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { @@ -1484,7 +1668,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } -void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, const CInv& inv, CConnman& connman) +void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& chainparams, const CInv& inv, CConnman& connman) { bool send = false; std::shared_ptr<const CBlock> a_recent_block; @@ -1624,16 +1808,18 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } } - // Trigger the peer node to send a getblocks request for the next batch of inventory - if (inv.hash == pfrom.hashContinue) { - // Send immediately. This must send even if redundant, - // and we want it right after the last block so they don't - // wait for other stuff first. - std::vector<CInv> vInv; - vInv.push_back(CInv(MSG_BLOCK, ::ChainActive().Tip()->GetBlockHash())); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); - pfrom.hashContinue.SetNull(); + LOCK(peer.m_block_inv_mutex); + // Trigger the peer node to send a getblocks request for the next batch of inventory + if (inv.hash == peer.m_continuation_block) { + // Send immediately. This must send even if redundant, + // and we want it right after the last block so they don't + // wait for other stuff first. + std::vector<CInv> vInv; + vInv.push_back(CInv(MSG_BLOCK, ::ChainActive().Tip()->GetBlockHash())); + connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + peer.m_continuation_block.SetNull(); + } } } } @@ -1733,7 +1919,7 @@ void static ProcessGetData(CNode& pfrom, Peer& peer, const CChainParams& chainpa if (it != peer.m_getdata_requests.end() && !pfrom.fPauseSend) { const CInv &inv = *it++; if (inv.IsGenBlkMsg()) { - ProcessGetBlockData(pfrom, chainparams, inv, connman); + ProcessGetBlockData(pfrom, peer, chainparams, inv, connman); } // else: If the first item on the queue is an unknown type, we erase it // and continue processing the queue on the next call. @@ -1768,7 +1954,8 @@ static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_ma return nFetchFlags; } -void PeerManager::SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req) { +void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req) +{ BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { @@ -1783,7 +1970,9 @@ void PeerManager::SendBlockTransactions(CNode& pfrom, const CBlock& block, const m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -void PeerManager::ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHeader>& headers, bool via_compact_block) +void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, + const std::vector<CBlockHeader>& headers, + bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); size_t nCount = headers.size(); @@ -1873,7 +2062,8 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHe // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), pfrom.nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); } @@ -1974,7 +2164,7 @@ void PeerManager::ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHe * may be added to if accepting an orphan causes its children to be * reconsidered. */ -void PeerManager::ProcessOrphanTx(std::set<uint256>& orphan_work_set) +void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) { AssertLockHeld(cs_main); AssertLockHeld(g_cs_orphans); @@ -2271,25 +2461,18 @@ static void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv, const CChainPar connman.PushMessage(&peer, std::move(msg)); } -void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - const std::chrono::microseconds time_received, - const std::atomic<bool>& interruptMsgProc) +void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, + const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); - if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) - { - LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); - return; - } PeerRef peer = GetPeerRef(pfrom.GetId()); if (peer == nullptr) return; if (msg_type == NetMsgType::VERSION) { - // Each connection can only send one version message - if (pfrom.nVersion != 0) - { - Misbehaving(pfrom.GetId(), 1, "redundant version message"); + if (pfrom.nVersion != 0) { + LogPrint(BCLog::NET, "redundant version message from peer=%d\n", pfrom.GetId()); return; } @@ -2301,7 +2484,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat ServiceFlags nServices; int nVersion; std::string cleanSubVer; - int nStartingHeight = -1; + int starting_height = -1; bool fRelay = true; vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; @@ -2332,7 +2515,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat cleanSubVer = SanitizeString(strSubVer); } if (!vRecv.empty()) { - vRecv >> nStartingHeight; + vRecv >> starting_height; } if (!vRecv.empty()) vRecv >> fRelay; @@ -2349,9 +2532,9 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat SeenLocal(addrMe); } - // Be shy and don't send version until we hear - if (pfrom.IsInboundConn()) - PushNodeVersion(pfrom, m_connman, GetAdjustedTime()); + // Inbound peers send us their version message when they connect. + // We send our version message in response. + if (pfrom.IsInboundConn()) PushNodeVersion(pfrom, GetAdjustedTime()); // Change version const int greatest_common_version = std::min(nVersion, PROTOCOL_VERSION); @@ -2364,10 +2547,16 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::WTXIDRELAY)); } - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - // Signal ADDRv2 support (BIP155). - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); + if (greatest_common_version >= 70016) { + // BIP155 defines addrv2 and sendaddrv2 for all protocol versions, but some + // implementations reject messages they don't know. As a courtesy, don't send + // it to nodes with a version before 70016, as no software is known to support + // BIP155 that doesn't announce at least that protocol version number. + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); + } + + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); @@ -2375,7 +2564,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LOCK(pfrom.cs_SubVer); pfrom.cleanSubVer = cleanSubVer; } - pfrom.nStartingHeight = nStartingHeight; + peer->m_starting_height = starting_height; // set nodes not relaying blocks and tx and not serving (parts) of the historical blockchain as "clients" pfrom.fClient = (!(nServices & NODE_NETWORK) && !(nServices & NODE_NETWORK_LIMITED)); @@ -2409,8 +2598,8 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // empty and no one will know who we are, so these mechanisms are // important to help us connect to the network. // - // We skip this for BLOCK_RELAY peers to avoid potentially leaking - // information about our BLOCK_RELAY connections via address relay. + // We skip this for block-relay-only peers to avoid potentially leaking + // information about our block-relay-only connections via address relay. if (fListen && !::ChainstateActive().IsInitialBlockDownload()) { CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); @@ -2455,7 +2644,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom.nVersion, - pfrom.nStartingHeight, addrMe.ToString(), pfrom.GetId(), + peer->m_starting_height, addrMe.ToString(), pfrom.GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -2477,7 +2666,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (pfrom.nVersion == 0) { // Must have a version message before anything else - Misbehaving(pfrom.GetId(), 1, "non-version message before version handshake"); + LogPrint(BCLog::NET, "non-version message before version handshake. Message \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; } @@ -2488,13 +2677,10 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (pfrom.fSuccessfullyConnected) return; if (!pfrom.IsInboundConn()) { - // Mark this node as currently connected, so we update its timestamp later. - LOCK(cs_main); - State(pfrom.GetId())->fCurrentlyConnected = true; LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s (%s)\n", - pfrom.nVersion.load(), pfrom.nStartingHeight, + pfrom.nVersion.load(), peer->m_starting_height, pfrom.GetId(), (fLogIPs ? strprintf(", peeraddr=%s", pfrom.addr.ToString()) : ""), - pfrom.m_tx_relay == nullptr ? "block-relay" : "full-relay"); + pfrom.ConnectionTypeAsString()); } if (pfrom.GetCommonVersion() >= SENDHEADERS_VERSION) { @@ -2521,8 +2707,41 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } - // Feature negotiation of wtxidrelay should happen between VERSION and - // VERACK, to avoid relay problems from switching after a connection is up + if (msg_type == NetMsgType::SENDHEADERS) { + LOCK(cs_main); + State(pfrom.GetId())->fPreferHeaders = true; + return; + } + + if (msg_type == NetMsgType::SENDCMPCT) { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 0; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1 || ((pfrom.GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { + LOCK(cs_main); + // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) + if (!State(pfrom.GetId())->fProvidesHeaderAndIDs) { + State(pfrom.GetId())->fProvidesHeaderAndIDs = true; + State(pfrom.GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; + } + if (State(pfrom.GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) { // ignore later version announces + State(pfrom.GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + // save whether peer selects us as BIP152 high-bandwidth peer + // (receiving sendcmpct(1) signals high-bandwidth, sendcmpct(0) low-bandwidth) + pfrom.m_bip152_highbandwidth_from = fAnnounceUsingCMPCTBLOCK; + } + if (!State(pfrom.GetId())->fSupportsDesiredCmpctVersion) { + if (pfrom.GetLocalServices() & NODE_WITNESS) + State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); + else + State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); + } + } + return; + } + + // Feature negotiation of wtxidrelay must happen between VERSION and VERACK + // to avoid relay problems from switching after a connection is up. if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { // Disconnect peers that send wtxidrelay message after VERACK; this @@ -2540,6 +2759,17 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } + if (msg_type == NetMsgType::SENDADDRV2) { + if (pfrom.fSuccessfullyConnected) { + // Disconnect peers that send SENDADDRV2 message after VERACK; this + // must be negotiated between VERSION and VERACK. + pfrom.fDisconnect = true; + return; + } + pfrom.m_wants_addrv2 = true; + return; + } + if (!pfrom.fSuccessfullyConnected) { LogPrint(BCLog::NET, "Unsupported message \"%s\" prior to verack from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; @@ -2593,7 +2823,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, m_connman); + RelayAddress(pfrom, addr, fReachable, m_connman); } // Do not store addresses outside our network if (fReachable) @@ -2607,40 +2837,6 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } - if (msg_type == NetMsgType::SENDADDRV2) { - pfrom.m_wants_addrv2 = true; - return; - } - - if (msg_type == NetMsgType::SENDHEADERS) { - LOCK(cs_main); - State(pfrom.GetId())->fPreferHeaders = true; - return; - } - - if (msg_type == NetMsgType::SENDCMPCT) { - bool fAnnounceUsingCMPCTBLOCK = false; - uint64_t nCMPCTBLOCKVersion = 0; - vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; - if (nCMPCTBLOCKVersion == 1 || ((pfrom.GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { - LOCK(cs_main); - // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) - if (!State(pfrom.GetId())->fProvidesHeaderAndIDs) { - State(pfrom.GetId())->fProvidesHeaderAndIDs = true; - State(pfrom.GetId())->fWantsCmpctWitness = nCMPCTBLOCKVersion == 2; - } - if (State(pfrom.GetId())->fWantsCmpctWitness == (nCMPCTBLOCKVersion == 2)) // ignore later version announces - State(pfrom.GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; - if (!State(pfrom.GetId())->fSupportsDesiredCmpctVersion) { - if (pfrom.GetLocalServices() & NODE_WITNESS) - State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); - else - State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); - } - } - return; - } - if (msg_type == NetMsgType::INV) { std::vector<CInv> vInv; vRecv >> vInv; @@ -2652,7 +2848,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // We won't accept tx inv's if we're in blocks-only mode, or this is a // block-relay-only peer - bool fBlocksOnly = !g_relay_txes || (pfrom.m_tx_relay == nullptr); + bool fBlocksOnly = m_ignore_incoming_txs || (pfrom.m_tx_relay == nullptr); // Allow peers with relay permission to send data other than blocks in blocks only mode if (pfrom.HasPermission(PF_RELAY)) { @@ -2794,13 +2990,12 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } - WITH_LOCK(pfrom.cs_inventory, pfrom.vInventoryBlockToSend.push_back(pindex->GetBlockHash())); - if (--nLimit <= 0) - { + WITH_LOCK(peer->m_block_inv_mutex, peer->m_blocks_for_inv_relay.push_back(pindex->GetBlockHash())); + if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll // trigger the peer to getblocks the next batch of inventory. LogPrint(BCLog::NET, " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); - pfrom.hashContinue = pindex->GetBlockHash(); + WITH_LOCK(peer->m_block_inv_mutex, {peer->m_continuation_block = pindex->GetBlockHash();}); break; } } @@ -2929,7 +3124,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // Stop processing the transaction early if // 1) We are in blocks only mode and peer has no relay permission // 2) This peer is a block-relay-only peer - if ((!g_relay_txes && !pfrom.HasPermission(PF_RELAY)) || (pfrom.m_tx_relay == nullptr)) + if ((m_ignore_incoming_txs && !pfrom.HasPermission(PF_RELAY)) || (pfrom.m_tx_relay == nullptr)) { LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom.GetId()); pfrom.fDisconnect = true; @@ -3324,7 +3519,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be disconnected/discouraged. - return ProcessHeadersMessage(pfrom, {cmpctblock.header}, /*via_compact_block=*/true); + return ProcessHeadersMessage(pfrom, *peer, {cmpctblock.header}, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -3467,7 +3662,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - return ProcessHeadersMessage(pfrom, headers, /*via_compact_block=*/false); + return ProcessHeadersMessage(pfrom, *peer, headers, /*via_compact_block=*/false); } if (msg_type == NetMsgType::BLOCK) @@ -3711,7 +3906,6 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { if (pfrom.m_tx_relay != nullptr) { - LOCK(pfrom.m_tx_relay->cs_feeFilter); pfrom.m_tx_relay->minFeeFilter = newFeeFilter; } LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom.GetId()); @@ -3755,7 +3949,7 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat return; } -bool PeerManager::MaybeDiscourageAndDisconnect(CNode& pnode) +bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode) { const NodeId peer_id{pnode.GetId()}; PeerRef peer = GetPeerRef(peer_id); @@ -3791,13 +3985,13 @@ bool PeerManager::MaybeDiscourageAndDisconnect(CNode& pnode) } // Normal case: Disconnect the peer and discourage all nodes sharing the address - LogPrintf("Disconnecting and discouraging peer %d!\n", peer_id); + LogPrint(BCLog::NET, "Disconnecting and discouraging peer %d!\n", peer_id); if (m_banman) m_banman->Discourage(pnode.addr); m_connman.DisconnectNode(pnode.addr); return true; } -bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) +bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) { bool fMoreWork = false; @@ -3872,7 +4066,7 @@ bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgP return fMoreWork; } -void PeerManager::ConsiderEviction(CNode& pto, int64_t time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, int64_t time_in_seconds) { AssertLockHeld(cs_main); @@ -3925,13 +4119,56 @@ void PeerManager::ConsiderEviction(CNode& pto, int64_t time_in_seconds) } } -void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) +void PeerManagerImpl::EvictExtraOutboundPeers(int64_t time_in_seconds) { - // Check whether we have too many outbound peers - int extra_peers = m_connman.GetExtraOutboundCount(); - if (extra_peers > 0) { - // If we have more outbound peers than we target, disconnect one. - // Pick the outbound peer that least recently announced + // If we have any extra block-relay-only peers, disconnect the youngest unless + // it's given us a block -- in which case, compare with the second-youngest, and + // out of those two, disconnect the peer who least recently gave us a block. + // The youngest block-relay-only peer would be the extra peer we connected + // to temporarily in order to sync our tip; see net.cpp. + // Note that we use higher nodeid as a measure for most recent connection. + if (m_connman.GetExtraBlockRelayCount() > 0) { + std::pair<NodeId, int64_t> youngest_peer{-1, 0}, next_youngest_peer{-1, 0}; + + m_connman.ForEachNode([&](CNode* pnode) { + if (!pnode->IsBlockOnlyConn() || pnode->fDisconnect) return; + if (pnode->GetId() > youngest_peer.first) { + next_youngest_peer = youngest_peer; + youngest_peer.first = pnode->GetId(); + youngest_peer.second = pnode->nLastBlockTime; + } + }); + NodeId to_disconnect = youngest_peer.first; + if (youngest_peer.second > next_youngest_peer.second) { + // Our newest block-relay-only peer gave us a block more recently; + // disconnect our second youngest. + to_disconnect = next_youngest_peer.first; + } + m_connman.ForNode(to_disconnect, [&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + AssertLockHeld(::cs_main); + // Make sure we're not getting a block right now, and that + // we've been connected long enough for this eviction to happen + // at all. + // Note that we only request blocks from a peer if we learn of a + // valid headers chain with at least as much work as our tip. + CNodeState *node_state = State(pnode->GetId()); + if (node_state == nullptr || + (time_in_seconds - pnode->nTimeConnected >= MINIMUM_CONNECT_TIME && node_state->nBlocksInFlight == 0)) { + pnode->fDisconnect = true; + LogPrint(BCLog::NET, "disconnecting extra block-relay-only peer=%d (last block received at time %d)\n", pnode->GetId(), pnode->nLastBlockTime); + return true; + } else { + LogPrint(BCLog::NET, "keeping block-relay-only peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", + pnode->GetId(), pnode->nTimeConnected, node_state->nBlocksInFlight); + } + return false; + }); + } + + // Check whether we have too many outbound-full-relay peers + if (m_connman.GetExtraFullOutboundCount() > 0) { + // If we have more outbound-full-relay peers than we target, disconnect one. + // Pick the outbound-full-relay peer that least recently announced // us a new block, with ties broken by choosing the more recent // connection (higher node id) NodeId worst_peer = -1; @@ -3940,14 +4177,13 @@ void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) m_connman.ForEachNode([&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - // Ignore non-outbound peers, or nodes marked for disconnect already - if (!pnode->IsOutboundOrBlockRelayConn() || pnode->fDisconnect) return; + // Only consider outbound-full-relay peers that are not already + // marked for disconnection + if (!pnode->IsFullOutboundConn() || pnode->fDisconnect) return; CNodeState *state = State(pnode->GetId()); if (state == nullptr) return; // shouldn't be possible, but just in case // Don't evict our protected peers if (state->m_chain_sync.m_protect) return; - // Don't evict our block-relay-only peers. - if (pnode->m_tx_relay == nullptr) return; if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { worst_peer = pnode->GetId(); oldest_block_announcement = state->m_last_block_announcement; @@ -3984,7 +4220,7 @@ void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) } } -void PeerManager::CheckForStaleTipAndEvictPeers() +void PeerManagerImpl::CheckForStaleTipAndEvictPeers() { LOCK(cs_main); @@ -4003,6 +4239,11 @@ void PeerManager::CheckForStaleTipAndEvictPeers() } m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; } + + if (!m_initial_sync_finished && CanDirectFetch(m_chainparams.GetConsensus())) { + m_connman.StartExtraBlockRelayPeers(); + m_initial_sync_finished = true; + } } namespace { @@ -4026,8 +4267,9 @@ public: }; } -bool PeerManager::SendMessages(CNode* pto) +bool PeerManagerImpl::SendMessages(CNode* pto) { + PeerRef peer = GetPeerRef(pto->GetId()); const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll @@ -4079,6 +4321,15 @@ bool PeerManager::SendMessages(CNode* pto) auto current_time = GetTime<std::chrono::microseconds>(); if (pto->RelayAddrsWithConn() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { + // If we've sent before, clear the bloom filter for the peer, so that our + // self-announcement will actually go out. + // This might be unnecessary if the bloom filter has already rolled + // over since our last self-announcement, but there is only a small + // bandwidth cost that we can incur by doing this (which happens + // once a day on average). + if (pto->m_next_local_addr_send != 0us) { + pto->m_addr_known->reset(); + } AdvertiseLocal(pto); pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } @@ -4144,7 +4395,7 @@ bool PeerManager::SendMessages(CNode* pto) got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256())); } } @@ -4160,11 +4411,11 @@ bool PeerManager::SendMessages(CNode* pto) // If no header would connect, or if we have too many // blocks, or if the peer doesn't want headers, just // add all to the inv queue. - LOCK(pto->cs_inventory); + LOCK(peer->m_block_inv_mutex); std::vector<CBlock> vHeaders; bool fRevertToInv = ((!state.fPreferHeaders && - (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || - pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + (!state.fPreferHeaderAndIDs || peer->m_blocks_for_headers_relay.size() > 1)) || + peer->m_blocks_for_headers_relay.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = nullptr; // last header queued for delivery ProcessBlockAvailability(pto->GetId()); // ensure pindexBestKnownBlock is up-to-date @@ -4173,7 +4424,7 @@ bool PeerManager::SendMessages(CNode* pto) // Try to find first header that our peer doesn't have, and // then send all headers past that one. If we come across any // headers that aren't on ::ChainActive(), give up. - for (const uint256 &hash : pto->vBlockHashesToAnnounce) { + for (const uint256& hash : peer->m_blocks_for_headers_relay) { const CBlockIndex* pindex = LookupBlockIndex(hash); assert(pindex); if (::ChainActive()[pindex->nHeight] != pindex) { @@ -4190,7 +4441,7 @@ bool PeerManager::SendMessages(CNode* pto) // which should be caught by the prior check), but one // way this could happen is by using invalidateblock / // reconsiderblock repeatedly on the tip, causing it to - // be added multiple times to vBlockHashesToAnnounce. + // be added multiple times to m_blocks_for_headers_relay. // Robustly deal with this rare situation by reverting // to an inv. fRevertToInv = true; @@ -4262,10 +4513,10 @@ bool PeerManager::SendMessages(CNode* pto) } if (fRevertToInv) { // If falling back to using an inv, just try to inv the tip. - // The last entry in vBlockHashesToAnnounce was our tip at some point + // The last entry in m_blocks_for_headers_relay was our tip at some point // in the past. - if (!pto->vBlockHashesToAnnounce.empty()) { - const uint256 &hashToAnnounce = pto->vBlockHashesToAnnounce.back(); + if (!peer->m_blocks_for_headers_relay.empty()) { + const uint256& hashToAnnounce = peer->m_blocks_for_headers_relay.back(); const CBlockIndex* pindex = LookupBlockIndex(hashToAnnounce); assert(pindex); @@ -4279,13 +4530,13 @@ bool PeerManager::SendMessages(CNode* pto) // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { - pto->vInventoryBlockToSend.push_back(hashToAnnounce); + peer->m_blocks_for_inv_relay.push_back(hashToAnnounce); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, pto->GetId(), hashToAnnounce.ToString()); } } } - pto->vBlockHashesToAnnounce.clear(); + peer->m_blocks_for_headers_relay.clear(); } // @@ -4293,18 +4544,18 @@ bool PeerManager::SendMessages(CNode* pto) // std::vector<CInv> vInv; { - LOCK(pto->cs_inventory); - vInv.reserve(std::max<size_t>(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); + LOCK(peer->m_block_inv_mutex); + vInv.reserve(std::max<size_t>(peer->m_blocks_for_inv_relay.size(), INVENTORY_BROADCAST_MAX)); // Add blocks - for (const uint256& hash : pto->vInventoryBlockToSend) { + for (const uint256& hash : peer->m_blocks_for_inv_relay) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } - pto->vInventoryBlockToSend.clear(); + peer->m_blocks_for_inv_relay.clear(); if (pto->m_tx_relay != nullptr) { LOCK(pto->m_tx_relay->cs_tx_inventory); @@ -4330,11 +4581,7 @@ bool PeerManager::SendMessages(CNode* pto) if (fSendTrickle && pto->m_tx_relay->fSendMempool) { auto vtxinfo = m_mempool.infoAll(); pto->m_tx_relay->fSendMempool = false; - CFeeRate filterrate; - { - LOCK(pto->m_tx_relay->cs_feeFilter); - filterrate = CFeeRate(pto->m_tx_relay->minFeeFilter); - } + const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; LOCK(pto->m_tx_relay->cs_filter); @@ -4368,11 +4615,7 @@ bool PeerManager::SendMessages(CNode* pto) for (std::set<uint256>::iterator it = pto->m_tx_relay->setInventoryTxToSend.begin(); it != pto->m_tx_relay->setInventoryTxToSend.end(); it++) { vInvTx.push_back(it); } - CFeeRate filterrate; - { - LOCK(pto->m_tx_relay->cs_feeFilter); - filterrate = CFeeRate(pto->m_tx_relay->minFeeFilter); - } + const CFeeRate filterrate{pto->m_tx_relay->minFeeFilter.load()}; // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); @@ -4533,7 +4776,7 @@ bool PeerManager::SendMessages(CNode* pto) } // - // Message: getdata (non-blocks) + // Message: getdata (transactions) // std::vector<std::pair<NodeId, GenTxid>> expired; auto requestable = m_txrequest.GetRequestable(pto->GetId(), current_time, &expired); diff --git a/src/net_processing.h b/src/net_processing.h index 87eee566de..eaa3b142a8 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -6,19 +6,13 @@ #ifndef BITCOIN_NET_PROCESSING_H #define BITCOIN_NET_PROCESSING_H -#include <consensus/params.h> #include <net.h> #include <sync.h> -#include <txrequest.h> #include <validationinterface.h> -class BlockTransactionsRequest; -class BlockValidationState; -class CBlockHeader; class CChainParams; class CTxMemPool; class ChainstateManager; -class TxValidationState; extern RecursiveMutex cs_main; extern RecursiveMutex g_cs_orphans; @@ -32,129 +26,48 @@ static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ static const int DISCOURAGEMENT_THRESHOLD{100}; -class PeerManager final : public CValidationInterface, public NetEventsInterface { -public: - PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, - CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); +struct CNodeStateStats { + int nSyncHeight = -1; + int nCommonHeight = -1; + int m_starting_height = -1; + std::vector<int> vHeightInFlight; +}; - /** - * Overridden from CValidationInterface. - */ - void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; - void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; - /** - * Overridden from CValidationInterface. - */ - void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; - /** - * Overridden from CValidationInterface. - */ - void BlockChecked(const CBlock& block, const BlockValidationState& state) override; - /** - * Overridden from CValidationInterface. - */ - void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) override; +class PeerManager : public CValidationInterface, public NetEventsInterface +{ +public: + static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool, + bool ignore_incoming_txs); + virtual ~PeerManager() { } - /** Initialize a peer by adding it to mapNodeState and pushing a message requesting its version */ - void InitializeNode(CNode* pnode) override; - /** Handle removal of a peer by updating various state and removing it from mapNodeState */ - void FinalizeNode(const CNode& node, bool& fUpdateConnectionTime) override; - /** - * Process protocol messages received from a given node - * - * @param[in] pfrom The node which we have received messages from. - * @param[in] interrupt Interrupt condition for processing threads - */ - bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override; - /** - * Send queued protocol messages to be sent to a give node. - * - * @param[in] pto The node which we are sending messages to. - * @return True if there is more work to be done - */ - bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); + /** Get statistics from node state */ + virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) = 0; - /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound */ - void CheckForStaleTipAndEvictPeers(); - /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ - void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ - void ReattemptInitialBroadcast(CScheduler& scheduler) const; + /** Whether this node ignores txs received over p2p. */ + virtual bool IgnoresIncomingTxs() = 0; - /** Process a single message from a peer. Public for fuzz testing */ - void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, - const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc); + /** Set the best height */ + virtual void SetBestHeight(int height) = 0; /** * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. * Public for unit testing. */ - void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message); + virtual void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) = 0; -private: /** - * Potentially mark a node discouraged based on the contents of a BlockValidationState object - * - * @param[in] via_compact_block this bool is passed in because net_processing should - * punish peers differently depending on whether the data was provided in a compact - * block message or not. If the compact block had a valid header, but contained invalid - * txs, the peer should not be punished. See BIP 152. - * - * @return Returns true if the peer was punished (probably disconnected) - */ - bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, - bool via_compact_block, const std::string& message = ""); - - /** - * Potentially disconnect and discourage a node based on the contents of a TxValidationState object - * - * @return Returns true if the peer was punished (probably disconnected) - */ - bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = ""); - - /** Maybe disconnect a peer and discourage future connections from its address. - * - * @param[in] pnode The node to check. - * @return True if the peer was marked for disconnection in this function + * Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound. + * Public for unit testing. */ - bool MaybeDiscourageAndDisconnect(CNode& pnode); - - void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); - /** Process a single headers message from a peer. */ - void ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHeader>& headers, bool via_compact_block); - - void SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req); - - /** Register with TxRequestTracker that an INV has been received from a - * peer. The announcement parameters are decided in PeerManager and then - * passed to TxRequestTracker. */ - void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + virtual void CheckForStaleTipAndEvictPeers() = 0; - const CChainParams& m_chainparams; - CConnman& m_connman; - /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ - BanMan* const m_banman; - ChainstateManager& m_chainman; - CTxMemPool& m_mempool; - TxRequestTracker m_txrequest GUARDED_BY(::cs_main); - - int64_t m_stale_tip_check_time; //!< Next time to check for stale tip -}; - -struct CNodeStateStats { - int m_misbehavior_score = 0; - int nSyncHeight = -1; - int nCommonHeight = -1; - std::vector<int> vHeightInFlight; + /** Process a single message from a peer. Public for fuzz testing */ + virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) = 0; }; -/** Get statistics from node state */ -bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); - /** Relay transaction to every node */ void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 35e9161f58..85e46fd373 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -437,6 +437,11 @@ bool CNetAddr::IsValid() const return false; } + // CJDNS addresses always start with 0xfc + if (IsCJDNS() && (m_addr[0] != 0xFC)) { + return false; + } + // documentation IPv6 address if (IsRFC3849()) return false; @@ -1063,15 +1068,24 @@ CSubNet::CSubNet(const CNetAddr& addr, const CNetAddr& mask) : CSubNet() CSubNet::CSubNet(const CNetAddr& addr) : CSubNet() { - valid = addr.IsIPv4() || addr.IsIPv6(); - if (!valid) { + switch (addr.m_net) { + case NET_IPV4: + case NET_IPV6: + valid = true; + assert(addr.m_addr.size() <= sizeof(netmask)); + memset(netmask, 0xFF, addr.m_addr.size()); + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + valid = true; + break; + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: return; } - assert(addr.m_addr.size() <= sizeof(netmask)); - - memset(netmask, 0xFF, addr.m_addr.size()); - network = addr; } @@ -1083,6 +1097,21 @@ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; + + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + case NET_INTERNAL: + return addr == network; + case NET_UNROUTABLE: + case NET_MAX: + return false; + } + assert(network.m_addr.size() == addr.m_addr.size()); for (size_t x = 0; x < addr.m_addr.size(); ++x) { if ((addr.m_addr[x] & netmask[x]) != network.m_addr[x]) { @@ -1094,18 +1123,35 @@ bool CSubNet::Match(const CNetAddr &addr) const std::string CSubNet::ToString() const { - assert(network.m_addr.size() <= sizeof(netmask)); + std::string suffix; - uint8_t cidr = 0; + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: { + assert(network.m_addr.size() <= sizeof(netmask)); - for (size_t i = 0; i < network.m_addr.size(); ++i) { - if (netmask[i] == 0x00) { - break; + uint8_t cidr = 0; + + for (size_t i = 0; i < network.m_addr.size(); ++i) { + if (netmask[i] == 0x00) { + break; + } + cidr += NetmaskBits(netmask[i]); } - cidr += NetmaskBits(netmask[i]); + + suffix = strprintf("/%u", cidr); + break; + } + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: + break; } - return network.ToString() + strprintf("/%u", cidr); + return network.ToString() + suffix; } bool CSubNet::IsValid() const @@ -1115,7 +1161,19 @@ bool CSubNet::IsValid() const bool CSubNet::SanityCheck() const { - if (!(network.IsIPv4() || network.IsIPv6())) return false; + switch (network.m_net) { + case NET_IPV4: + case NET_IPV6: + break; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + return true; + case NET_INTERNAL: + case NET_UNROUTABLE: + case NET_MAX: + return false; + } for (size_t x = 0; x < network.m_addr.size(); ++x) { if (network.m_addr[x] & ~netmask[x]) return false; diff --git a/src/netaddress.h b/src/netaddress.h index 29b2eaafeb..b9eade7fd5 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -462,11 +462,33 @@ class CSubNet bool SanityCheck() const; public: + /** + * Construct an invalid subnet (empty, `Match()` always returns false). + */ CSubNet(); + + /** + * Construct from a given network start and number of bits (CIDR mask). + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask CIDR mask, must be in [0, 32] for IPv4 addresses and in [0, 128] for + * IPv6 addresses. Otherwise an invalid subnet is created. + */ CSubNet(const CNetAddr& addr, uint8_t mask); + + /** + * Construct from a given network start and mask. + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask Network mask, must be of the same type as `addr` and not contain 0-bits + * followed by 1-bits. Otherwise an invalid subnet is created. + */ CSubNet(const CNetAddr& addr, const CNetAddr& mask); - //constructor for single ip subnet (<ipv4>/32 or <ipv6>/128) + /** + * Construct a single-host subnet. + * @param[in] addr The sole address to be contained in the subnet, can also be non-IPv[46]. + */ explicit CSubNet(const CNetAddr& addr); bool Match(const CNetAddr &addr) const; @@ -483,7 +505,7 @@ class CSubNet READWRITE(obj.network); if (obj.network.IsIPv4()) { // Before commit 102867c587f5f7954232fb8ed8e85cda78bb4d32, CSubNet used the last 4 bytes of netmask - // to store the relevant bytes for an IPv4 mask. For compatiblity reasons, keep doing so in + // to store the relevant bytes for an IPv4 mask. For compatibility reasons, keep doing so in // serialized form. unsigned char dummy[12] = {0}; READWRITE(dummy); diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index ffb3fe2f29..89fb4758f9 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index fb46ea1731..02e50c4dbe 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -112,7 +112,7 @@ bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, CoinStatsHashType hash_t } // The legacy hash serializes the hashBlock -static void PrepareHash(CHashWriter& ss, CCoinsStats& stats) +static void PrepareHash(CHashWriter& ss, const CCoinsStats& stats) { ss << stats.hashBlock; } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 2a7441c10e..7c56bfc2ad 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.cpp b/src/node/context.cpp index 49d0c37235..958221a913 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <net.h> #include <net_processing.h> +#include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 3228831ed1..9b611bf8f5 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -12,6 +12,7 @@ class ArgsManager; class BanMan; +class CBlockPolicyEstimator; class CConnman; class CScheduler; class CTxMemPool; @@ -36,6 +37,7 @@ class WalletClient; struct NodeContext { std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; + std::unique_ptr<CBlockPolicyEstimator> fee_estimator; std::unique_ptr<PeerManager> peerman; ChainstateManager* chainman{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<BanMan> banman; diff --git a/src/interfaces/chain.cpp b/src/node/interfaces.cpp index 4c5ebe66fc..e07eaa33d8 100644 --- a/src/interfaces/chain.cpp +++ b/src/node/interfaces.cpp @@ -2,18 +2,26 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <interfaces/chain.h> - +#include <addrdb.h> +#include <banman.h> +#include <boost/signals2/signal.hpp> #include <chain.h> #include <chainparams.h> +#include <init.h> +#include <interfaces/chain.h> #include <interfaces/handler.h> +#include <interfaces/node.h> #include <interfaces/wallet.h> +#include <mapport.h> #include <net.h> #include <net_processing.h> +#include <netaddress.h> +#include <netbase.h> #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> #include <node/ui_interface.h> +#include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -23,22 +31,274 @@ #include <rpc/protocol.h> #include <rpc/server.h> #include <shutdown.h> +#include <support/allocators/secure.h> #include <sync.h> #include <timedata.h> #include <txmempool.h> #include <uint256.h> #include <univalue.h> +#include <util/check.h> +#include <util/ref.h> #include <util/system.h> +#include <util/translation.h> #include <validation.h> #include <validationinterface.h> +#include <warnings.h> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif #include <memory> #include <utility> -namespace interfaces { +using interfaces::BlockTip; +using interfaces::Chain; +using interfaces::FoundBlock; +using interfaces::Handler; +using interfaces::MakeHandler; +using interfaces::Node; +using interfaces::WalletClient; + +namespace node { namespace { +class NodeImpl : public Node +{ +public: + explicit NodeImpl(NodeContext* context) { setContext(context); } + void initLogging() override { InitLogging(*Assert(m_context->args)); } + void initParameterInteraction() override { InitParameterInteraction(*Assert(m_context->args)); } + bilingual_str getWarnings() override { return GetWarnings(true); } + uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } + bool baseInitialize() override + { + return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && + AppInitLockDataDirectory() && AppInitInterfaces(*m_context); + } + bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override + { + return AppInitMain(m_context_ref, *m_context, tip_info); + } + void appShutdown() override + { + Interrupt(*m_context); + Shutdown(*m_context); + } + void startShutdown() override + { + StartShutdown(); + // Stop RPC for clean shutdown if any of waitfor* commands is executed. + if (gArgs.GetBoolArg("-server", false)) { + InterruptRPC(); + StopRPC(); + } + } + bool shutdownRequested() override { return ShutdownRequested(); } + void mapPort(bool use_upnp, bool use_natpmp) override { StartMapPort(use_upnp, use_natpmp); } + bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } + size_t getNodeCount(CConnman::NumConnections flags) override + { + return m_context->connman ? m_context->connman->GetNodeCount(flags) : 0; + } + bool getNodesStats(NodesStats& stats) override + { + stats.clear(); + + if (m_context->connman) { + std::vector<CNodeStats> stats_temp; + m_context->connman->GetNodeStats(stats_temp); + + stats.reserve(stats_temp.size()); + for (auto& node_stats_temp : stats_temp) { + stats.emplace_back(std::move(node_stats_temp), false, CNodeStateStats()); + } + + // Try to retrieve the CNodeStateStats for each node. + if (m_context->peerman) { + TRY_LOCK(::cs_main, lockMain); + if (lockMain) { + for (auto& node_stats : stats) { + std::get<1>(node_stats) = + m_context->peerman->GetNodeStateStats(std::get<0>(node_stats).nodeid, std::get<2>(node_stats)); + } + } + } + return true; + } + return false; + } + bool getBanned(banmap_t& banmap) override + { + if (m_context->banman) { + m_context->banman->GetBanned(banmap); + return true; + } + return false; + } + bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override + { + if (m_context->banman) { + m_context->banman->Ban(net_addr, ban_time_offset); + return true; + } + return false; + } + bool unban(const CSubNet& ip) override + { + if (m_context->banman) { + m_context->banman->Unban(ip); + return true; + } + return false; + } + bool disconnectByAddress(const CNetAddr& net_addr) override + { + if (m_context->connman) { + return m_context->connman->DisconnectNode(net_addr); + } + return false; + } + bool disconnectById(NodeId id) override + { + if (m_context->connman) { + return m_context->connman->DisconnectNode(id); + } + return false; + } + int64_t getTotalBytesRecv() override { return m_context->connman ? m_context->connman->GetTotalBytesRecv() : 0; } + int64_t getTotalBytesSent() override { return m_context->connman ? m_context->connman->GetTotalBytesSent() : 0; } + size_t getMempoolSize() override { return m_context->mempool ? m_context->mempool->size() : 0; } + size_t getMempoolDynamicUsage() override { return m_context->mempool ? m_context->mempool->DynamicMemoryUsage() : 0; } + bool getHeaderTip(int& height, int64_t& block_time) override + { + LOCK(::cs_main); + if (::pindexBestHeader) { + height = ::pindexBestHeader->nHeight; + block_time = ::pindexBestHeader->GetBlockTime(); + return true; + } + return false; + } + int getNumBlocks() override + { + LOCK(::cs_main); + return ::ChainActive().Height(); + } + uint256 getBestBlockHash() override + { + const CBlockIndex* tip = WITH_LOCK(::cs_main, return ::ChainActive().Tip()); + return tip ? tip->GetBlockHash() : Params().GenesisBlock().GetHash(); + } + int64_t getLastBlockTime() override + { + LOCK(::cs_main); + if (::ChainActive().Tip()) { + return ::ChainActive().Tip()->GetBlockTime(); + } + return Params().GenesisBlock().GetBlockTime(); // Genesis block's time of current network + } + double getVerificationProgress() override + { + const CBlockIndex* tip; + { + LOCK(::cs_main); + tip = ::ChainActive().Tip(); + } + return GuessVerificationProgress(Params().TxData(), tip); + } + bool isInitialBlockDownload() override { return ::ChainstateActive().IsInitialBlockDownload(); } + bool getReindex() override { return ::fReindex; } + bool getImporting() override { return ::fImporting; } + void setNetworkActive(bool active) override + { + if (m_context->connman) { + m_context->connman->SetNetworkActive(active); + } + } + bool getNetworkActive() override { return m_context->connman && m_context->connman->GetNetworkActive(); } + CFeeRate getDustRelayFee() override { return ::dustRelayFee; } + UniValue executeRpc(const std::string& command, const UniValue& params, const std::string& uri) override + { + JSONRPCRequest req(m_context_ref); + req.params = params; + req.strMethod = command; + req.URI = uri; + return ::tableRPC.execute(req); + } + std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); } + void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } + void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } + bool getUnspentOutput(const COutPoint& output, Coin& coin) override + { + LOCK(::cs_main); + return ::ChainstateActive().CoinsTip().GetCoin(output, coin); + } + WalletClient& walletClient() override + { + return *Assert(m_context->wallet_client); + } + std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) override + { + return MakeHandler(::uiInterface.InitMessage_connect(fn)); + } + std::unique_ptr<Handler> handleMessageBox(MessageBoxFn fn) override + { + return MakeHandler(::uiInterface.ThreadSafeMessageBox_connect(fn)); + } + std::unique_ptr<Handler> handleQuestion(QuestionFn fn) override + { + return MakeHandler(::uiInterface.ThreadSafeQuestion_connect(fn)); + } + std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override + { + return MakeHandler(::uiInterface.ShowProgress_connect(fn)); + } + std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override + { + return MakeHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); + } + std::unique_ptr<Handler> handleNotifyNetworkActiveChanged(NotifyNetworkActiveChangedFn fn) override + { + return MakeHandler(::uiInterface.NotifyNetworkActiveChanged_connect(fn)); + } + std::unique_ptr<Handler> handleNotifyAlertChanged(NotifyAlertChangedFn fn) override + { + return MakeHandler(::uiInterface.NotifyAlertChanged_connect(fn)); + } + std::unique_ptr<Handler> handleBannedListChanged(BannedListChangedFn fn) override + { + return MakeHandler(::uiInterface.BannedListChanged_connect(fn)); + } + std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override + { + return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, + GuessVerificationProgress(Params().TxData(), block)); + })); + } + std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override + { + return MakeHandler( + ::uiInterface.NotifyHeaderTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, + /* verification progress is unused when a header was received */ 0); + })); + } + NodeContext* context() override { return m_context; } + void setContext(NodeContext* context) override + { + m_context = context; + if (context) { + m_context_ref.Set(*context); + } else { + m_context_ref.Clear(); + } + } + NodeContext* m_context{nullptr}; + util::Ref m_context_ref; +}; -bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock) +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active) { if (!index) return false; if (block.m_hash) *block.m_hash = index->GetBlockHash(); @@ -46,6 +306,8 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec if (block.m_time) *block.m_time = index->GetBlockTime(); if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); + if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; + if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); if (block.m_data) { REVERSE_LOCK(lock); if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); @@ -148,48 +410,33 @@ public: Optional<int> getHeight() override { LOCK(::cs_main); - int height = ::ChainActive().Height(); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + int height = active.Height(); if (height >= 0) { return height; } return nullopt; } - Optional<int> getBlockHeight(const uint256& hash) override - { - LOCK(::cs_main); - CBlockIndex* block = LookupBlockIndex(hash); - if (block && ::ChainActive().Contains(block)) { - return block->nHeight; - } - return nullopt; - } uint256 getBlockHash(int height) override { LOCK(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + CBlockIndex* block = active[height]; assert(block); return block->GetBlockHash(); } bool haveBlockOnDisk(int height) override { LOCK(cs_main); - CBlockIndex* block = ::ChainActive()[height]; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + CBlockIndex* block = active[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } - Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override - { - LOCK(cs_main); - CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); - if (block) { - if (hash) *hash = block->GetBlockHash(); - return block->nHeight; - } - return nullopt; - } CBlockLocator getTipLocator() override { LOCK(cs_main); - return ::ChainActive().GetLocator(); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + return active.GetLocator(); } bool checkFinalTx(const CTransaction& tx) override { @@ -199,7 +446,8 @@ public: Optional<int> findLocatorFork(const CBlockLocator& locator) override { LOCK(cs_main); - if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + if (CBlockIndex* fork = FindForkInGlobalIndex(active, locator)) { return fork->nHeight; } return nullopt; @@ -207,47 +455,45 @@ public: bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - return FillBlock(LookupBlockIndex(hash), block, lock); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + return FillBlock(LookupBlockIndex(hash), block, lock, active); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - return FillBlock(ChainActive().FindEarliestAtLeast(min_time, min_height), block, lock); - } - bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next, bool* reorg) override { - WAIT_LOCK(cs_main, lock); - CBlockIndex* block = ChainActive()[block_height]; - if (block && block->GetBlockHash() != block_hash) block = nullptr; - if (reorg) *reorg = !block; - return FillBlock(block ? ChainActive()[block_height + 1] : nullptr, next, lock); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active); } bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override { WAIT_LOCK(cs_main, lock); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { - return FillBlock(ancestor, ancestor_out, lock); + return FillBlock(ancestor, ancestor_out, lock, active); } } - return FillBlock(nullptr, ancestor_out, lock); + return FillBlock(nullptr, ancestor_out, lock, active); } bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override { WAIT_LOCK(cs_main, lock); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); const CBlockIndex* block = LookupBlockIndex(block_hash); const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; - return FillBlock(ancestor, ancestor_out, lock); + return FillBlock(ancestor, ancestor_out, lock, active); } bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override { WAIT_LOCK(cs_main, lock); + const CChain& active = Assert(m_node.chainman)->ActiveChain(); const CBlockIndex* block1 = LookupBlockIndex(block_hash1); const CBlockIndex* block2 = LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. - return FillBlock(ancestor, ancestor_out, lock) & FillBlock(block1, block1_out, lock) & FillBlock(block2, block2_out, lock); + return FillBlock(ancestor, ancestor_out, lock, active) & FillBlock(block1, block1_out, lock, active) & FillBlock(block2, block2_out, lock, active); } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -327,11 +573,13 @@ public: } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { - return ::feeEstimator.estimateSmartFee(num_blocks, calc, conservative); + if (!m_node.fee_estimator) return {}; + return m_node.fee_estimator->estimateSmartFee(num_blocks, calc, conservative); } unsigned int estimateMaxBlocks() override { - return ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + if (!m_node.fee_estimator) return 0; + return m_node.fee_estimator->HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); } CFeeRate mempoolMinFee() override { @@ -365,7 +613,8 @@ public: { if (!old_tip.IsNull()) { LOCK(::cs_main); - if (old_tip == ::ChainActive().Tip()->GetBlockHash()) return; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); + if (old_tip == active.Tip()->GetBlockHash()) return; } SyncWithValidationInterfaceQueue(); } @@ -411,7 +660,9 @@ public: NodeContext& m_node; }; } // namespace +} // namespace node -std::unique_ptr<Chain> MakeChain(NodeContext& node) { return MakeUnique<ChainImpl>(node); } - +namespace interfaces { +std::unique_ptr<Node> MakeNode(NodeContext* context) { return MakeUnique<node::NodeImpl>(context); } +std::unique_ptr<Chain> MakeChain(NodeContext& context) { return MakeUnique<node::ChainImpl>(context); } } // namespace interfaces diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 97d5aad8e4..d3bb9687a8 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/transaction.h b/src/node/transaction.h index 6491700d44..0c016ff04e 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -36,6 +36,6 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; * @param[in] wait_callback wait until callbacks have been processed to avoid stale result due to a sequentially RPC. * return error */ -NODISCARD TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); +[[nodiscard]] TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index c8b4d60fd0..fe78cb46bd 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/noui.cpp b/src/noui.cpp index 3c82512fac..6fe5c5638c 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/noui.h b/src/noui.h index 8ec5708328..cf273ffb33 100644 --- a/src/noui.h +++ b/src/noui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/optional.h b/src/optional.h index a382cd7b77..583c56eabd 100644 --- a/src/optional.h +++ b/src/optional.h @@ -1,26 +1,20 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_OPTIONAL_H #define BITCOIN_OPTIONAL_H +#include <optional> #include <utility> -#include <boost/optional.hpp> - //! Substitute for C++17 std::optional +//! DEPRECATED use std::optional in new code. template <typename T> -using Optional = boost::optional<T>; - -//! Substitute for C++17 std::make_optional -template <typename T> -Optional<T> MakeOptional(bool condition, T&& value) -{ - return boost::make_optional(condition, std::forward<T>(value)); -} +using Optional = std::optional<T>; //! Substitute for C++17 std::nullopt -static auto& nullopt = boost::none; +//! DEPRECATED use std::nullopt in new code. +static auto& nullopt = std::nullopt; #endif // BITCOIN_OPTIONAL_H diff --git a/src/outputtype.cpp b/src/outputtype.cpp index e978852826..d96fb282c5 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -19,8 +19,6 @@ static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; -const std::array<OutputType, 3> OUTPUT_TYPES = {OutputType::LEGACY, OutputType::P2SH_SEGWIT, OutputType::BECH32}; - bool ParseOutputType(const std::string& type, OutputType& output_type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { diff --git a/src/outputtype.h b/src/outputtype.h index 77a16b1d05..88422e5824 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,9 +20,13 @@ enum class OutputType { BECH32, }; -extern const std::array<OutputType, 3> OUTPUT_TYPES; +static constexpr auto OUTPUT_TYPES = std::array{ + OutputType::LEGACY, + OutputType::P2SH_SEGWIT, + OutputType::BECH32, +}; -NODISCARD bool ParseOutputType(const std::string& str, OutputType& output_type); +[[nodiscard]] bool ParseOutputType(const std::string& str, OutputType& output_type); const std::string& FormatOutputType(OutputType type); /** diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index 04e0e117a5..3da85fedf9 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 7c5660ac8a..86ae507957 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 0f31093dbb..7da171d2e1 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,28 +1,29 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/fees.h> #include <clientversion.h> +#include <fs.h> +#include <logging.h> #include <streams.h> #include <txmempool.h> #include <util/system.h> -static constexpr double INF_FEERATE = 1e99; - -std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) { - static const std::map<FeeEstimateHorizon, std::string> horizon_strings = { - {FeeEstimateHorizon::SHORT_HALFLIFE, "short"}, - {FeeEstimateHorizon::MED_HALFLIFE, "medium"}, - {FeeEstimateHorizon::LONG_HALFLIFE, "long"}, - }; - auto horizon_string = horizon_strings.find(horizon); +static const char* FEE_ESTIMATES_FILENAME = "fee_estimates.dat"; - if (horizon_string == horizon_strings.end()) return "unknown"; +static constexpr double INF_FEERATE = 1e99; - return horizon_string->second; +std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon) +{ + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: return "short"; + case FeeEstimateHorizon::MED_HALFLIFE: return "medium"; + case FeeEstimateHorizon::LONG_HALFLIFE: return "long"; + } // no default case, so the compiler can warn about missing cases + assert(false); } /** @@ -489,6 +490,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { buckets.push_back(bucketBoundary); bucketMap[bucketBoundary] = bucketIndex; @@ -500,6 +502,13 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + + // If the fee estimation file is present, read recorded estimations + fs::path est_filepath = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_file(fsbridge::fopen(est_filepath, "rb"), SER_DISK, CLIENT_VERSION); + if (est_file.IsNull() || !Read(est_file)) { + LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", est_filepath.string()); + } } CBlockPolicyEstimator::~CBlockPolicyEstimator() @@ -632,7 +641,7 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const { - TxConfirmStats* stats; + TxConfirmStats* stats = nullptr; double sufficientTxs = SUFFICIENT_FEETXS; switch (horizon) { case FeeEstimateHorizon::SHORT_HALFLIFE: { @@ -648,10 +657,8 @@ CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThr stats = longStats.get(); break; } - default: { - throw std::out_of_range("CBlockPolicyEstimator::estimateRawFee unknown FeeEstimateHorizon"); - } - } + } // no default case, so the compiler can warn about missing cases + assert(stats); LOCK(m_cs_fee_estimator); // Return failure if trying to analyze a target we're not tracking @@ -681,10 +688,8 @@ unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon hori case FeeEstimateHorizon::LONG_HALFLIFE: { return longStats->GetMaxConfirms(); } - default: { - throw std::out_of_range("CBlockPolicyEstimator::HighestTargetTracked unknown FeeEstimateHorizon"); - } - } + } // no default case, so the compiler can warn about missing cases + assert(false); } unsigned int CBlockPolicyEstimator::BlockSpan() const @@ -856,6 +861,15 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation return CFeeRate(llround(median)); } +void CBlockPolicyEstimator::Flush() { + FlushUnconfirmed(); + + fs::path est_filepath = GetDataDir() / FEE_ESTIMATES_FILENAME; + CAutoFile est_file(fsbridge::fopen(est_filepath, "wb"), SER_DISK, CLIENT_VERSION); + if (est_file.IsNull() || !Write(est_file)) { + LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", est_filepath.string()); + } +} bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { @@ -888,8 +902,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) LOCK(m_cs_fee_estimator); int nVersionRequired, nVersionThatWrote; filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + if (nVersionRequired > CLIENT_VERSION) { + throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired)); + } // Read fee estimates file into temporary variables so existing data // structures aren't corrupted if there is an exception. @@ -907,8 +922,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) std::vector<double> fileBuckets; filein >> fileBuckets; size_t numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) + if (numBuckets <= 1 || numBuckets > 1000) { throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + } std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); diff --git a/src/policy/fees.h b/src/policy/fees.h index 8ea8816dc3..c444d71a31 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -11,6 +11,7 @@ #include <random.h> #include <sync.h> +#include <array> #include <map> #include <memory> #include <string> @@ -25,9 +26,15 @@ class TxConfirmStats; /* Identifier for each of the 3 different TxConfirmStats which will track * history over different time horizons. */ enum class FeeEstimateHorizon { - SHORT_HALFLIFE = 0, - MED_HALFLIFE = 1, - LONG_HALFLIFE = 2 + SHORT_HALFLIFE, + MED_HALFLIFE, + LONG_HALFLIFE, +}; + +static constexpr auto ALL_FEE_ESTIMATE_HORIZONS = std::array{ + FeeEstimateHorizon::SHORT_HALFLIFE, + FeeEstimateHorizon::MED_HALFLIFE, + FeeEstimateHorizon::LONG_HALFLIFE, }; std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon); @@ -215,6 +222,9 @@ public: /** Calculation of highest target that estimates are tracked for */ unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const; + /** Drop still unconfirmed transactions and record current estimations, if the fee estimation file is present. */ + void Flush(); + private: mutable RecursiveMutex m_cs_fee_estimator; diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 91997aa883..2b5fd4179d 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,7 +75,7 @@ bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType) bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason) { - if (tx.nVersion > CTransaction::MAX_STANDARD_VERSION || tx.nVersion < 1) { + if (tx.nVersion > TX_MAX_STANDARD_VERSION || tx.nVersion < 1) { reason = "version"; return false; } @@ -92,14 +92,15 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR for (const CTxIn& txin : tx.vin) { - // Biggest 'standard' txin is a 15-of-15 P2SH multisig with compressed - // keys (remember the 520 byte limit on redeemScript size). That works - // out to a (15*(33+1))+3=513 byte redeemScript, 513+1+15*(73+1)+3=1627 - // bytes of scriptSig, which we round off to 1650 bytes for some minor - // future-proofing. That's also enough to spend a 20-of-20 - // CHECKMULTISIG scriptPubKey, though such a scriptPubKey is not - // considered standard. - if (txin.scriptSig.size() > 1650) { + // Biggest 'standard' txin involving only keys is a 15-of-15 P2SH + // multisig with compressed keys (remember the 520 byte limit on + // redeemScript size). That works out to a (15*(33+1))+3=513 byte + // redeemScript, 513+1+15*(73+1)+3=1627 bytes of scriptSig, which + // we round off to 1650(MAX_STANDARD_SCRIPTSIG_SIZE) bytes for + // some minor future-proofing. That's also enough to spend a + // 20-of-20 CHECKMULTISIG scriptPubKey, though such a scriptPubKey + // is not considered standard. + if (txin.scriptSig.size() > MAX_STANDARD_SCRIPTSIG_SIZE) { reason = "scriptsig-size"; return false; } diff --git a/src/policy/policy.h b/src/policy/policy.h index 8090dff4c6..f2a3f35546 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,12 +38,14 @@ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_PERMIT_BAREMULTISIG = true; /** The maximum number of witness stack items in a standard P2WSH script */ static const unsigned int MAX_STANDARD_P2WSH_STACK_ITEMS = 100; -/** The maximum size of each witness stack item in a standard P2WSH script */ +/** The maximum size in bytes of each witness stack item in a standard P2WSH script */ static const unsigned int MAX_STANDARD_P2WSH_STACK_ITEM_SIZE = 80; -/** The maximum size of each witness stack item in a standard BIP 342 script (Taproot, leaf version 0xc0) */ +/** The maximum size in bytes of each witness stack item in a standard BIP 342 script (Taproot, leaf version 0xc0) */ static const unsigned int MAX_STANDARD_TAPSCRIPT_STACK_ITEM_SIZE = 80; -/** The maximum size of a standard witnessScript */ +/** The maximum size in bytes of a standard witnessScript */ static const unsigned int MAX_STANDARD_P2WSH_SCRIPT_SIZE = 3600; +/** The maximum size of a standard ScriptSig */ +static const unsigned int MAX_STANDARD_SCRIPTSIG_SIZE = 1650; /** Min feerate for defining dust. Historically this has been based on the * minRelayTxFee, however changing the dust limit changes which transactions are * standard and should be done with care and ideally rarely. It makes sense to @@ -88,23 +90,32 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsStandard(const CScript& scriptPubKey, TxoutType& whichType); - /** - * Check for standard transaction types - * @return True if all outputs (scriptPubKeys) use only standard transaction forms - */ + + +// Changing the default transaction version requires a two step process: first +// adapting relay policy by bumping TX_MAX_STANDARD_VERSION, and then later +// allowing the new transaction version in the wallet/RPC. +static constexpr decltype(CTransaction::nVersion) TX_MAX_STANDARD_VERSION{2}; + +/** +* Check for standard transaction types +* @return True if all outputs (scriptPubKeys) use only standard transaction forms +*/ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeRate& dust_relay_fee, std::string& reason); - /** - * Check for standard transaction types - * @param[in] mapInputs Map of previous transactions that have outputs we're spending - * @param[in] taproot_active Whether or taproot consensus rules are active (used to decide whether spends of them are permitted) - * @return True if all inputs (scriptSigs) use only standard transaction forms - */ +/** +* Check for standard transaction types +* @param[in] mapInputs Map of previous transactions that have outputs we're spending +* @param[in] taproot_active Whether or taproot consensus rules are active (used to decide whether spends of them are permitted) +* @return True if all inputs (scriptSigs) use only standard transaction forms +*/ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, bool taproot_active); - /** - * Check if the transaction is over standard P2WSH resources limit: - * 3600bytes witnessScript size, 80bytes per witness stack element, 100 witness stack elements - * These limits are adequate for multi-signature up to n-of-100 using OP_CHECKSIG, OP_ADD, and OP_EQUAL, - */ +/** +* Check if the transaction is over standard P2WSH resources limit: +* 3600bytes witnessScript size, 80bytes per witness stack element, 100 witness stack elements +* These limits are adequate for multisignatures up to n-of-100 using OP_CHECKSIG, OP_ADD, and OP_EQUAL. +* +* Also enforce a maximum stack item size limit and no annexes for tapscript spends. +*/ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs); /** Compute the virtual transaction size (weight reinterpreted as bytes). */ diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 4b55934891..8125b41c41 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.h b/src/policy/rbf.h index f84e6e5286..e078070c1c 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/block.h b/src/primitives/block.h index fd8fc8b868..2d10853607 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index e6183cf2f4..245206b906 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -77,8 +77,6 @@ uint256 CTransaction::ComputeWitnessHash() const return SerializeHash(*this, SER_GETHASH, 0); } -/* For backward compatibility, the hash is initialized to 0. TODO: remove the need for this default constructor entirely. */ -CTransaction::CTransaction() : vin(), vout(), nVersion(CTransaction::CURRENT_VERSION), nLockTime(0), hash{}, m_witness_hash{} {} CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 00544f64fe..6bf36ee854 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -262,12 +262,6 @@ public: // Default transaction version. static const int32_t CURRENT_VERSION=2; - // Changing the default transaction version requires a two step process: first - // adapting relay policy by bumping MAX_STANDARD_VERSION, and then later date - // bumping the default CURRENT_VERSION at which point both CURRENT_VERSION and - // MAX_STANDARD_VERSION will be equal. - static const int32_t MAX_STANDARD_VERSION=2; - // The local variables are made const to prevent unintended modification // without updating the cached hash value. However, CTransaction is not // actually immutable; deserialization and assignment are implemented, @@ -287,12 +281,9 @@ private: uint256 ComputeWitnessHash() const; public: - /** Construct a CTransaction that qualifies as IsNull() */ - CTransaction(); - /** Convert a CMutableTransaction into a CTransaction. */ - explicit CTransaction(const CMutableTransaction &tx); - CTransaction(CMutableTransaction &&tx); + explicit CTransaction(const CMutableTransaction& tx); + CTransaction(CMutableTransaction&& tx); template <typename Stream> inline void Serialize(Stream& s) const { @@ -393,7 +384,6 @@ struct CMutableTransaction }; typedef std::shared_ptr<const CTransaction> CTransactionRef; -static inline CTransactionRef MakeTransactionRef() { return std::make_shared<const CTransaction>(); } template <typename Tx> static inline CTransactionRef MakeTransactionRef(Tx&& txIn) { return std::make_shared<const CTransaction>(std::forward<Tx>(txIn)); } /** A generic txid reference (txid or wtxid). */ diff --git a/src/protocol.cpp b/src/protocol.cpp index dc8f795a0c..56e738eaa8 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -203,7 +203,6 @@ static std::string serviceFlagToStr(size_t bit) switch ((ServiceFlags)service_flag) { case NODE_NONE: abort(); // impossible case NODE_NETWORK: return "NETWORK"; - case NODE_GETUTXO: return "GETUTXO"; case NODE_BLOOM: return "BLOOM"; case NODE_WITNESS: return "WITNESS"; case NODE_COMPACT_FILTERS: return "COMPACT_FILTERS"; diff --git a/src/protocol.h b/src/protocol.h index 309fac621c..f183db0501 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -273,10 +273,6 @@ enum ServiceFlags : uint64_t { // NODE_NETWORK means that the node is capable of serving the complete block chain. It is currently // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), - // NODE_GETUTXO means the node is capable of responding to the getutxo protocol request. - // Bitcoin Core does not support this but a patch set called Bitcoin XT does. - // See BIP 64 for details on how this is implemented. - NODE_GETUTXO = (1 << 1), // NODE_BLOOM means the node is capable and willing to handle bloom-filtered connections. // Bitcoin Core nodes used to support this by default, without advertising this bit, // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) diff --git a/src/psbt.cpp b/src/psbt.cpp index 3fb743e5db..4db57d3cd0 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -3,6 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <psbt.h> + +#include <util/check.h> #include <util/strencodings.h> @@ -207,7 +209,8 @@ size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt) { void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index) { - const CTxOut& out = psbt.tx->vout.at(index); + CMutableTransaction& tx = *Assert(psbt.tx); + const CTxOut& out = tx.vout.at(index); PSBTOutput& psbt_out = psbt.outputs.at(index); // Fill a SignatureData with output info @@ -217,7 +220,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio // Construct a would-be spend of this output, to update sigdata with. // Note that ProduceSignature is used to fill in metadata (not actual signatures), // so provider does not need to provide any private keys (it can be a HidingSigningProvider). - MutableTransactionSignatureCreator creator(psbt.tx.get_ptr(), /* index */ 0, out.nValue, SIGHASH_ALL); + MutableTransactionSignatureCreator creator(&tx, /* index */ 0, out.nValue, SIGHASH_ALL); ProduceSignature(provider, creator, out.scriptPubKey, sigdata); // Put redeem_script, witness_script, key paths, into PSBTOutput. diff --git a/src/psbt.h b/src/psbt.h index 0951b76f83..b566726ee3 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -398,7 +398,7 @@ struct PartiallySignedTransaction /** Merge psbt into this. The two psbts must have the same underlying CTransaction (i.e. the * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ - NODISCARD bool Merge(const PartiallySignedTransaction& psbt); + [[nodiscard]] bool Merge(const PartiallySignedTransaction& psbt); bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} @@ -605,11 +605,11 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti * @param[in] psbtxs the PSBTs to combine * @return error (OK if we successfully combined the transactions, other error if they were not compatible) */ -NODISCARD TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); +[[nodiscard]] TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); //! Decode a base64ed PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); +[[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); //! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); +[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); #endif // BITCOIN_PSBT_H diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 4d734fc891..fc1624af25 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/pubkey.h b/src/pubkey.h index 0f784b86e4..12514fc3c9 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -170,6 +170,15 @@ public: /* * Check syntactic correctness. * + * When setting a pubkey (Set()) or deserializing fails (its header bytes + * don't match the length of the data), the size is set to 0. Thus, + * by checking size, one can observe whether Set() or deserialization has + * failed. + * + * This does not check for more than that. In particular, it does not verify + * that the coordinates correspond to a point on the curve (see IsFullyValid() + * for that instead). + * * Note that this is consensus critical as CheckECDSASignature() calls it! */ bool IsValid() const @@ -214,7 +223,7 @@ private: public: /** Construct an x-only pubkey from exactly 32 bytes. */ - XOnlyPubKey(Span<const unsigned char> bytes); + explicit XOnlyPubKey(Span<const unsigned char> bytes); /** Verify a Schnorr signature against this public key. * diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index aa4ec04497..0fa52359a8 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 3d303a6f68..c6a364ccbd 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 665c8e6053..bb444f22b3 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -11,7 +11,6 @@ #include <wallet/wallet.h> #include <algorithm> -#include <typeinfo> #include <QFont> #include <QDebug> @@ -82,8 +81,9 @@ public: { for (const auto& address : wallet.getAddresses()) { - if (pk_hash_only && address.dest.type() != typeid(PKHash)) + if (pk_hash_only && !std::holds_alternative<PKHash>(address.dest)) { continue; + } AddressTableEntry::Type addressType = translateTransactionType( QString::fromStdString(address.purpose), address.is_mine); cachedAddressTable.append(AddressTableEntry(addressType, @@ -177,13 +177,17 @@ AddressTableModel::~AddressTableModel() int AddressTableModel::rowCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return priv->size(); } int AddressTableModel::columnCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return columns.length(); } @@ -257,7 +261,7 @@ bool AddressTableModel::setData(const QModelIndex &index, const QVariant &value, } else if(index.column() == Address) { CTxDestination newAddress = DecodeDestination(value.toString().toStdString()); // Refuse to set invalid address, set error status and return false - if(boost::get<CNoDestination>(&newAddress)) + if(std::get_if<CNoDestination>(&newAddress)) { editStatus = INVALID_ADDRESS; return false; diff --git a/src/qt/addresstablemodel.h b/src/qt/addresstablemodel.h index 73316cadc4..6cc14654ef 100644 --- a/src/qt/addresstablemodel.h +++ b/src/qt/addresstablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 3d1963b6e6..c2349b78f9 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -58,14 +58,6 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureStri ui->passEdit3->hide(); setWindowTitle(tr("Unlock wallet")); break; - case Decrypt: // Ask passphrase - ui->warningLabel->setText(tr("This operation needs your wallet passphrase to decrypt the wallet.")); - ui->passLabel2->hide(); - ui->passEdit2->hide(); - ui->passLabel3->hide(); - ui->passEdit3->hide(); - setWindowTitle(tr("Decrypt wallet")); - break; case ChangePass: // Ask old passphrase + new passphrase x2 setWindowTitle(tr("Change passphrase")); ui->warningLabel->setText(tr("Enter the old passphrase and new passphrase for the wallet.")); @@ -133,8 +125,7 @@ void AskPassphraseDialog::accept() "</b></qt>"); } else { assert(model != nullptr); - if(model->setWalletEncrypted(true, newpass1)) - { + if (model->setWalletEncrypted(newpass1)) { QMessageBox::warning(this, tr("Wallet encrypted"), "<qt>" + tr("Your wallet is now encrypted. ") + encryption_reminder + @@ -144,9 +135,7 @@ void AskPassphraseDialog::accept() "For security reasons, previous backups of the unencrypted wallet file " "will become useless as soon as you start using the new, encrypted wallet.") + "</b></qt>"); - } - else - { + } else { QMessageBox::critical(this, tr("Wallet encryption failed"), tr("Wallet encryption failed due to an internal error. Your wallet was not encrypted.")); } @@ -176,17 +165,6 @@ void AskPassphraseDialog::accept() QMessageBox::critical(this, tr("Wallet unlock failed"), e.what()); } break; - case Decrypt: - if(!model->setWalletEncrypted(false, oldpass)) - { - QMessageBox::critical(this, tr("Wallet decryption failed"), - tr("The passphrase entered for the wallet decryption was incorrect.")); - } - else - { - QDialog::accept(); // Success - } - break; case ChangePass: if(newpass1 == newpass2) { @@ -221,7 +199,6 @@ void AskPassphraseDialog::textChanged() acceptable = !ui->passEdit2->text().isEmpty() && !ui->passEdit3->text().isEmpty(); break; case Unlock: // Old passphrase x1 - case Decrypt: acceptable = !ui->passEdit1->text().isEmpty(); break; case ChangePass: // Old passphrase x1, new passphrase x2 diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 9557e72936..66031b08d3 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,7 +26,6 @@ public: Encrypt, /**< Ask passphrase twice and encrypt */ Unlock, /**< Ask passphrase and unlock */ ChangePass, /**< Ask old passphrase + new passphrase twice */ - Decrypt /**< Ask passphrase and decrypt wallet */ }; explicit AskPassphraseDialog(Mode mode, QWidget *parent, SecureString* passphrase_out = nullptr); diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 2739b21a9d..a01a7bc386 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -11,6 +11,7 @@ #include <QDateTime> #include <QList> +#include <QLocale> #include <QModelIndex> #include <QVariant> @@ -97,13 +98,17 @@ BanTableModel::~BanTableModel() int BanTableModel::rowCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return priv->size(); } int BanTableModel::columnCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return columns.length(); } @@ -122,7 +127,7 @@ QVariant BanTableModel::data(const QModelIndex &index, int role) const case Bantime: QDateTime date = QDateTime::fromMSecsSinceEpoch(0); date = date.addSecs(rec->banEntry.nBanUntil); - return date.toString(Qt::SystemLocaleLongDate); + return QLocale::system().toString(date, QLocale::LongFormat); } } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 63b4107f7e..8efb0e35d0 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -77,7 +77,7 @@ static void RegisterMetaTypes() #ifdef ENABLE_WALLET qRegisterMetaType<WalletModel*>(); #endif - // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // Register typedefs (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType) // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) qRegisterMetaType<CAmount>("CAmount"); qRegisterMetaType<size_t>("size_t"); @@ -263,7 +263,7 @@ void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) { assert(!m_splash); - m_splash = new SplashScreen(nullptr, networkStyle); + m_splash = new SplashScreen(networkStyle); // We don't hold a direct pointer to the splash screen after creation, but the splash // screen will take care of deleting itself when finish() happens. m_splash->show(); diff --git a/src/qt/bitcoinaddressvalidator.h b/src/qt/bitcoinaddressvalidator.h index 52c06828a3..ae698c72a6 100644 --- a/src/qt/bitcoinaddressvalidator.h +++ b/src/qt/bitcoinaddressvalidator.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index a953c991bc..2af3502be3 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index d3e61aac29..c60d9a2c90 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 23370e6ad3..f2a49e5a76 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -615,10 +615,10 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel, interfaces::BlockAndH OptionsModel* optionsModel = _clientModel->getOptionsModel(); if (optionsModel && trayIcon) { // be aware of the tray icon disable state change reported by the OptionsModel object. - connect(optionsModel, &OptionsModel::hideTrayIconChanged, this, &BitcoinGUI::setTrayIconVisible); + connect(optionsModel, &OptionsModel::showTrayIconChanged, trayIcon, &QSystemTrayIcon::setVisible); // initialize the disable state of the tray icon with the current value in the model. - setTrayIconVisible(optionsModel->getHideTrayIcon()); + trayIcon->setVisible(optionsModel->getShowTrayIcon()); } } else { // Disable possibility to show main window via action @@ -1311,7 +1311,7 @@ void BitcoinGUI::updateProxyIcon() bool proxy_enabled = clientModel->getProxyInfo(ip_port); if (proxy_enabled) { - if (labelProxyIcon->pixmap() == nullptr) { + if (!GUIUtil::HasPixmap(labelProxyIcon)) { QString ip_port_q = QString::fromStdString(ip_port); labelProxyIcon->setPixmap(platformStyle->SingleColorIcon(":/icons/proxy").pixmap(STATUSBAR_ICONSIZE, STATUSBAR_ICONSIZE)); labelProxyIcon->setToolTip(tr("Proxy is <b>enabled</b>: %1").arg(ip_port_q)); @@ -1387,14 +1387,6 @@ void BitcoinGUI::showProgress(const QString &title, int nProgress) } } -void BitcoinGUI::setTrayIconVisible(bool fHideTrayIcon) -{ - if (trayIcon) - { - trayIcon->setVisible(!fHideTrayIcon); - } -} - void BitcoinGUI::showModalOverlay() { if (modalOverlay && (progressBar->isVisible() || modalOverlay->isLayerVisible())) diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 912297a74e..147f19e68d 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -49,6 +49,7 @@ struct BlockAndHeaderTipInfo; QT_BEGIN_NAMESPACE class QAction; class QComboBox; +class QDateTime; class QMenu; class QProgressBar; class QProgressDialog; @@ -317,9 +318,6 @@ public Q_SLOTS: /** Show progress dialog e.g. for verifychain */ void showProgress(const QString &title, int nProgress); - /** When hideTrayIcon setting is changed in OptionsModel hide or show the icon accordingly. */ - void setTrayIconVisible(bool); - void showModalOverlay(); }; diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index fd55c547fc..9660ba99f7 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,8 @@ #include <cassert> +static constexpr auto MAX_DIGITS_BTC = 16; + BitcoinUnits::BitcoinUnits(QObject *parent): QAbstractListModel(parent), unitlist(availableUnits()) @@ -108,7 +110,9 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator qint64 n_abs = (n > 0 ? n : -n); qint64 quotient = n_abs / coin; QString quotient_str = QString::number(quotient); - if (justify) quotient_str = quotient_str.rightJustified(16 - num_decimals, ' '); + if (justify) { + quotient_str = quotient_str.rightJustified(MAX_DIGITS_BTC - num_decimals, ' '); + } // Use SI-style thin space separators as these are locale independent and can't be // confused with the decimal marker. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7f12cce1d9..7ac4cc040b 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 7c72858501..08abef7866 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -455,7 +455,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * else if(ExtractDestination(out.txout.scriptPubKey, address)) { CPubKey pubkey; - PKHash *pkhash = boost::get<PKHash>(&address); + PKHash* pkhash = std::get_if<PKHash>(&address); if (pkhash && model->wallet().getPubKey(out.txout.scriptPubKey, ToKeyID(*pkhash), pubkey)) { nBytesInputs += (pubkey.IsCompressed() ? 148 : 180); diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 3de7fd6d54..6ceb3de61d 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 2ded6a1d89..70f080f4de 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,12 +51,15 @@ CreateWalletDialog::CreateWalletDialog(QWidget* parent) : } }); - #ifndef USE_SQLITE +#ifndef USE_SQLITE ui->descriptor_checkbox->setToolTip(tr("Compiled without sqlite support (required for descriptor wallets)")); ui->descriptor_checkbox->setEnabled(false); ui->descriptor_checkbox->setChecked(false); - #endif - +#endif +#ifndef USE_BDB + ui->descriptor_checkbox->setEnabled(false); + ui->descriptor_checkbox->setChecked(true); +#endif } CreateWalletDialog::~CreateWalletDialog() diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index e0af9a20a8..cd25f856a2 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/editaddressdialog.h b/src/qt/editaddressdialog.h index 3be63156fd..f7ad80bb2d 100644 --- a/src/qt/editaddressdialog.h +++ b/src/qt/editaddressdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index ea713e1abd..0b33c2cb8d 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -60,7 +60,7 @@ <rect> <x>20</x> <y>50</y> - <width>171</width> + <width>220</width> <height>22</height> </rect> </property> @@ -79,7 +79,7 @@ <rect> <x>20</x> <y>90</y> - <width>130</width> + <width>220</width> <height>21</height> </rect> </property> @@ -98,7 +98,7 @@ <rect> <x>20</x> <y>115</y> - <width>171</width> + <width>220</width> <height>22</height> </rect> </property> @@ -130,7 +130,7 @@ <rect> <x>20</x> <y>155</y> - <width>171</width> + <width>220</width> <height>22</height> </rect> </property> diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index d210faec03..bce578a158 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -109,39 +109,13 @@ </widget> </item> <item row="3" column="0"> - <widget class="QLabel" name="label_berkeleyDBVersion"> - <property name="text"> - <string>Using BerkeleyDB version</string> - </property> - <property name="indent"> - <number>10</number> - </property> - </widget> - </item> - <item row="3" column="1" colspan="2"> - <widget class="QLabel" name="berkeleyDBVersion"> - <property name="cursor"> - <cursorShape>IBeamCursor</cursorShape> - </property> - <property name="text"> - <string>N/A</string> - </property> - <property name="textFormat"> - <enum>Qt::PlainText</enum> - </property> - <property name="textInteractionFlags"> - <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> - </property> - </widget> - </item> - <item row="4" column="0"> <widget class="QLabel" name="label_12"> <property name="text"> <string>Datadir</string> </property> </widget> </item> - <item row="4" column="1" colspan="2"> + <item row="3" column="1" colspan="2"> <widget class="QLabel" name="dataDir"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -163,14 +137,14 @@ </property> </widget> </item> - <item row="5" column="0"> + <item row="4" column="0"> <widget class="QLabel" name="label_11"> <property name="text"> <string>Blocksdir</string> </property> </widget> </item> - <item row="5" column="1" colspan="2"> + <item row="4" column="1" colspan="2"> <widget class="QLabel" name="blocksDir"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -192,14 +166,14 @@ </property> </widget> </item> - <item row="6" column="0"> + <item row="5" column="0"> <widget class="QLabel" name="label_13"> <property name="text"> <string>Startup time</string> </property> </widget> </item> - <item row="6" column="1" colspan="2"> + <item row="5" column="1" colspan="2"> <widget class="QLabel" name="startupTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -215,7 +189,7 @@ </property> </widget> </item> - <item row="7" column="0"> + <item row="6" column="0"> <widget class="QLabel" name="labelNetwork"> <property name="font"> <font> @@ -228,14 +202,14 @@ </property> </widget> </item> - <item row="8" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="label_8"> <property name="text"> <string>Name</string> </property> </widget> </item> - <item row="8" column="1" colspan="2"> + <item row="7" column="1" colspan="2"> <widget class="QLabel" name="networkName"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -251,14 +225,14 @@ </property> </widget> </item> - <item row="9" column="0"> + <item row="8" column="0"> <widget class="QLabel" name="label_7"> <property name="text"> <string>Number of connections</string> </property> </widget> </item> - <item row="9" column="1" colspan="2"> + <item row="8" column="1" colspan="2"> <widget class="QLabel" name="numberOfConnections"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -274,7 +248,7 @@ </property> </widget> </item> - <item row="10" column="0"> + <item row="9" column="0"> <widget class="QLabel" name="label_10"> <property name="font"> <font> @@ -287,14 +261,14 @@ </property> </widget> </item> - <item row="11" column="0"> + <item row="10" column="0"> <widget class="QLabel" name="label_3"> <property name="text"> <string>Current block height</string> </property> </widget> </item> - <item row="11" column="1" colspan="2"> + <item row="10" column="1" colspan="2"> <widget class="QLabel" name="numberOfBlocks"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -310,14 +284,14 @@ </property> </widget> </item> - <item row="12" column="0"> + <item row="11" column="0"> <widget class="QLabel" name="labelLastBlockTime"> <property name="text"> <string>Last block time</string> </property> </widget> </item> - <item row="12" column="1" colspan="2"> + <item row="11" column="1" colspan="2"> <widget class="QLabel" name="lastBlockTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -333,7 +307,7 @@ </property> </widget> </item> - <item row="13" column="0"> + <item row="12" column="0"> <widget class="QLabel" name="labelMempoolTitle"> <property name="font"> <font> @@ -346,14 +320,14 @@ </property> </widget> </item> - <item row="14" column="0"> + <item row="13" column="0"> <widget class="QLabel" name="labelNumberOfTransactions"> <property name="text"> <string>Current number of transactions</string> </property> </widget> </item> - <item row="14" column="1"> + <item row="13" column="1"> <widget class="QLabel" name="mempoolNumberTxs"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -369,14 +343,14 @@ </property> </widget> </item> - <item row="15" column="0"> + <item row="14" column="0"> <widget class="QLabel" name="labelMemoryUsage"> <property name="text"> <string>Memory usage</string> </property> </widget> </item> - <item row="15" column="1"> + <item row="14" column="1"> <widget class="QLabel" name="mempoolSize"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -392,7 +366,7 @@ </property> </widget> </item> - <item row="13" column="2" rowspan="3"> + <item row="12" column="2" rowspan="3"> <layout class="QVBoxLayout" name="verticalLayoutDebugButton"> <property name="spacing"> <number>3</number> @@ -432,7 +406,7 @@ </item> </layout> </item> - <item row="16" column="0"> + <item row="15" column="0"> <spacer name="verticalSpacer"> <property name="orientation"> <enum>Qt::Vertical</enum> @@ -1103,14 +1077,17 @@ </widget> </item> <item row="1" column="0"> - <widget class="QLabel" name="label_23"> + <widget class="QLabel" name="peerConnectionTypeLabel"> + <property name="toolTip"> + <string>The type of peer connection:<ul><li>Inbound: initiated by peer</li><li>Outbound Full Relay: default</li><li>Outbound Block Relay: does not relay transactions or addresses</li><li>Outbound Manual: added using RPC %1 or %2/%3 configuration options</li><li>Outbound Feeler: short-lived, for testing addresses</li><li>Outbound Address Fetch: short-lived, for soliciting addresses</li></ul></string> + </property> <property name="text"> - <string>Direction</string> + <string>Connection Type</string> </property> </widget> </item> <item row="1" column="1"> - <widget class="QLabel" name="peerDirection"> + <widget class="QLabel" name="peerConnectionType"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> </property> @@ -1126,13 +1103,39 @@ </widget> </item> <item row="2" column="0"> + <widget class="QLabel" name="peerNetworkLabel"> + <property name="toolTip"> + <string>The network protocol this peer is connected through: IPv4, IPv6, Onion, I2P, or CJDNS.</string> + </property> + <property name="text"> + <string>Network</string> + </property> + </widget> + </item> + <item row="2" column="1"> + <widget class="QLabel" name="peerNetwork"> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>N/A</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + <property name="textInteractionFlags"> + <set>Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set> + </property> + </widget> + </item> + <item row="3" column="0"> <widget class="QLabel" name="label_21"> <property name="text"> <string>Version</string> </property> </widget> </item> - <item row="2" column="1"> + <item row="3" column="1"> <widget class="QLabel" name="peerVersion"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1148,14 +1151,14 @@ </property> </widget> </item> - <item row="3" column="0"> + <item row="4" column="0"> <widget class="QLabel" name="label_28"> <property name="text"> <string>User Agent</string> </property> </widget> </item> - <item row="3" column="1"> + <item row="4" column="1"> <widget class="QLabel" name="peerSubversion"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1171,14 +1174,14 @@ </property> </widget> </item> - <item row="4" column="0"> + <item row="5" column="0"> <widget class="QLabel" name="label_4"> <property name="text"> <string>Services</string> </property> </widget> </item> - <item row="4" column="1"> + <item row="5" column="1"> <widget class="QLabel" name="peerServices"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1194,14 +1197,14 @@ </property> </widget> </item> - <item row="5" column="0"> + <item row="6" column="0"> <widget class="QLabel" name="label_29"> <property name="text"> <string>Starting Block</string> </property> </widget> </item> - <item row="5" column="1"> + <item row="6" column="1"> <widget class="QLabel" name="peerHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1217,14 +1220,14 @@ </property> </widget> </item> - <item row="6" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="label_27"> <property name="text"> <string>Synced Headers</string> </property> </widget> </item> - <item row="6" column="1"> + <item row="7" column="1"> <widget class="QLabel" name="peerSyncHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1240,14 +1243,14 @@ </property> </widget> </item> - <item row="7" column="0"> + <item row="8" column="0"> <widget class="QLabel" name="label_25"> <property name="text"> <string>Synced Blocks</string> </property> </widget> </item> - <item row="7" column="1"> + <item row="8" column="1"> <widget class="QLabel" name="peerCommonHeight"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1263,14 +1266,14 @@ </property> </widget> </item> - <item row="8" column="0"> + <item row="9" column="0"> <widget class="QLabel" name="label_22"> <property name="text"> <string>Connection Time</string> </property> </widget> </item> - <item row="8" column="1"> + <item row="9" column="1"> <widget class="QLabel" name="peerConnTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1286,14 +1289,14 @@ </property> </widget> </item> - <item row="9" column="0"> + <item row="10" column="0"> <widget class="QLabel" name="label_15"> <property name="text"> <string>Last Send</string> </property> </widget> </item> - <item row="9" column="1"> + <item row="10" column="1"> <widget class="QLabel" name="peerLastSend"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1309,14 +1312,14 @@ </property> </widget> </item> - <item row="10" column="0"> + <item row="11" column="0"> <widget class="QLabel" name="label_19"> <property name="text"> <string>Last Receive</string> </property> </widget> </item> - <item row="10" column="1"> + <item row="11" column="1"> <widget class="QLabel" name="peerLastRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1332,14 +1335,14 @@ </property> </widget> </item> - <item row="11" column="0"> + <item row="12" column="0"> <widget class="QLabel" name="label_18"> <property name="text"> <string>Sent</string> </property> </widget> </item> - <item row="11" column="1"> + <item row="12" column="1"> <widget class="QLabel" name="peerBytesSent"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1355,14 +1358,14 @@ </property> </widget> </item> - <item row="12" column="0"> + <item row="13" column="0"> <widget class="QLabel" name="label_20"> <property name="text"> <string>Received</string> </property> </widget> </item> - <item row="12" column="1"> + <item row="13" column="1"> <widget class="QLabel" name="peerBytesRecv"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1378,14 +1381,14 @@ </property> </widget> </item> - <item row="13" column="0"> + <item row="14" column="0"> <widget class="QLabel" name="label_26"> <property name="text"> <string>Ping Time</string> </property> </widget> </item> - <item row="13" column="1"> + <item row="14" column="1"> <widget class="QLabel" name="peerPingTime"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1401,7 +1404,7 @@ </property> </widget> </item> - <item row="14" column="0"> + <item row="15" column="0"> <widget class="QLabel" name="peerPingWaitLabel"> <property name="toolTip"> <string>The duration of a currently outstanding ping.</string> @@ -1411,7 +1414,7 @@ </property> </widget> </item> - <item row="14" column="1"> + <item row="15" column="1"> <widget class="QLabel" name="peerPingWait"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1427,14 +1430,14 @@ </property> </widget> </item> - <item row="15" column="0"> + <item row="16" column="0"> <widget class="QLabel" name="peerMinPingLabel"> <property name="text"> <string>Min Ping</string> </property> </widget> </item> - <item row="15" column="1"> + <item row="16" column="1"> <widget class="QLabel" name="peerMinPing"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1450,14 +1453,14 @@ </property> </widget> </item> - <item row="16" column="0"> + <item row="17" column="0"> <widget class="QLabel" name="label_timeoffset"> <property name="text"> <string>Time Offset</string> </property> </widget> </item> - <item row="16" column="1"> + <item row="17" column="1"> <widget class="QLabel" name="timeoffset"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1473,7 +1476,7 @@ </property> </widget> </item> - <item row="17" column="0"> + <item row="18" column="0"> <widget class="QLabel" name="peerMappedASLabel"> <property name="toolTip"> <string>The mapped Autonomous System used for diversifying peer selection.</string> @@ -1483,7 +1486,7 @@ </property> </widget> </item> - <item row="17" column="1"> + <item row="18" column="1"> <widget class="QLabel" name="peerMappedAS"> <property name="cursor"> <cursorShape>IBeamCursor</cursorShape> @@ -1499,7 +1502,7 @@ </property> </widget> </item> - <item row="18" column="0"> + <item row="19" column="0"> <spacer name="verticalSpacer_3"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 0016fb9739..8181cc47e2 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -55,7 +55,7 @@ <item> <widget class="QCheckBox" name="prune"> <property name="toolTip"> - <string>Disables some advanced features but all blocks will still be fully validated. Reverting this setting requires re-downloading the entire blockchain. Actual disk usage may be somewhat higher.</string> + <string>Enabling pruning significantly reduces the disk space required to store transactions. All blocks are still fully validated. Reverting this setting requires re-downloading the entire blockchain.</string> </property> <property name="text"> <string>Prune &block storage to</string> @@ -260,6 +260,16 @@ </widget> </item> <item> + <widget class="QCheckBox" name="mapPortNatpmp"> + <property name="toolTip"> + <string>Automatically open the Bitcoin client port on the router. This only works when your router supports NAT-PMP and it is enabled. The external port could be random.</string> + </property> + <property name="text"> + <string>Map port using NA&T-PMP</string> + </property> + </widget> + </item> + <item> <widget class="QCheckBox" name="allowIncoming"> <property name="toolTip"> <string>Accept connections from outside.</string> @@ -568,12 +578,15 @@ </attribute> <layout class="QVBoxLayout" name="verticalLayout_Window"> <item> - <widget class="QCheckBox" name="hideTrayIcon"> + <widget class="QCheckBox" name="showTrayIcon"> <property name="toolTip"> - <string>Hide the icon from the system tray.</string> + <string>Show the icon in the system tray.</string> </property> <property name="text"> - <string>&Hide tray icon</string> + <string>&Show tray icon</string> + </property> + <property name="checked"> + <bool>true</bool> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index bab17562a6..430ecd322f 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -43,6 +43,7 @@ #include <QKeyEvent> #include <QLineEdit> #include <QList> +#include <QLocale> #include <QMenu> #include <QMouseEvent> #include <QProgressDialog> @@ -67,7 +68,7 @@ namespace GUIUtil { QString dateTimeStr(const QDateTime &date) { - return date.date().toString(Qt::SystemLocaleShortDate) + QString(" ") + date.toString("hh:mm"); + return QLocale::system().toString(date.date(), QLocale::ShortFormat) + QString(" ") + date.toString("hh:mm"); } QString dateTimeStr(qint64 nTime) @@ -654,7 +655,7 @@ bool SetStartOnSystemStartup(bool fAutoStart) #elif defined(Q_OS_LINUX) // Follow the Desktop Application Autostart Spec: -// http://standards.freedesktop.org/autostart-spec/autostart-spec-latest.html +// https://specifications.freedesktop.org/autostart-spec/autostart-spec-latest.html fs::path static GetAutostartDir() { @@ -748,6 +749,34 @@ QString boostPathToQString(const fs::path &path) return QString::fromStdString(path.string()); } +QString NetworkToQString(Network net) +{ + switch (net) { + case NET_UNROUTABLE: return QObject::tr("Unroutable"); + case NET_IPV4: return "IPv4"; + case NET_IPV6: return "IPv6"; + case NET_ONION: return "Onion"; + case NET_I2P: return "I2P"; + case NET_CJDNS: return "CJDNS"; + case NET_INTERNAL: return QObject::tr("Internal"); + case NET_MAX: assert(false); + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +QString ConnectionTypeToQString(ConnectionType conn_type) +{ + switch (conn_type) { + case ConnectionType::INBOUND: return QObject::tr("Inbound"); + case ConnectionType::OUTBOUND_FULL_RELAY: return QObject::tr("Outbound Full Relay"); + case ConnectionType::BLOCK_RELAY: return QObject::tr("Outbound Block Relay"); + case ConnectionType::MANUAL: return QObject::tr("Outbound Manual"); + case ConnectionType::FEELER: return QObject::tr("Outbound Feeler"); + case ConnectionType::ADDR_FETCH: return QObject::tr("Outbound Address Fetch"); + } // no default case, so the compiler can warn about missing cases + assert(false); +} + QString formatDurationStr(int secs) { QStringList strList; @@ -920,4 +949,35 @@ void PopupMenu(QMenu* menu, const QPoint& point, QAction* at_action) menu->popup(point, at_action); } +QDateTime StartOfDay(const QDate& date) +{ +#if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0)) + return date.startOfDay(); +#else + return QDateTime(date); +#endif +} + +bool HasPixmap(const QLabel* label) +{ +#if (QT_VERSION >= QT_VERSION_CHECK(5, 15, 0)) + return !label->pixmap(Qt::ReturnByValue).isNull(); +#else + return label->pixmap() != nullptr; +#endif +} + +QImage GetImage(const QLabel* label) +{ + if (!HasPixmap(label)) { + return QImage(); + } + +#if (QT_VERSION >= QT_VERSION_CHECK(5, 15, 0)) + return label->pixmap(Qt::ReturnByValue).toImage(); +#else + return label->pixmap()->toImage(); +#endif +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 2bd94b5eb3..c471b888f7 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -7,16 +7,18 @@ #include <amount.h> #include <fs.h> +#include <net.h> +#include <netaddress.h> #include <QEvent> #include <QHeaderView> #include <QItemDelegate> +#include <QLabel> #include <QMessageBox> #include <QObject> #include <QProgressBar> #include <QString> #include <QTableView> -#include <QLabel> class QValidatedLineEdit; class SendCoinsRecipient; @@ -218,22 +220,28 @@ namespace GUIUtil bool GetStartOnSystemStartup(); bool SetStartOnSystemStartup(bool fAutoStart); - /* Convert QString to OS specific boost path through UTF-8 */ + /** Convert QString to OS specific boost path through UTF-8 */ fs::path qstringToBoostPath(const QString &path); - /* Convert OS specific boost path to QString through UTF-8 */ + /** Convert OS specific boost path to QString through UTF-8 */ QString boostPathToQString(const fs::path &path); - /* Convert seconds into a QString with days, hours, mins, secs */ + /** Convert enum Network to QString */ + QString NetworkToQString(Network net); + + /** Convert enum ConnectionType to QString */ + QString ConnectionTypeToQString(ConnectionType conn_type); + + /** Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); - /* Format CNodeStats.nServices bitmask into a user-readable string */ + /** Format CNodeStats.nServices bitmask into a user-readable string */ QString formatServicesStr(quint64 mask); - /* Format a CNodeStats.m_ping_usec into a user-readable string or display N/A, if 0*/ + /** Format a CNodeStats.m_ping_usec into a user-readable string or display N/A, if 0 */ QString formatPingTime(int64_t ping_usec); - /* Format a CNodeCombinedStats.nTimeOffset into a user-readable string. */ + /** Format a CNodeCombinedStats.nTimeOffset into a user-readable string */ QString formatTimeOffset(int64_t nTimeOffset); QString formatNiceTimeOffset(qint64 secs); @@ -289,7 +297,7 @@ namespace GUIUtil /** * Returns the distance in pixels appropriate for drawing a subsequent character after text. * - * In Qt 5.12 and before the QFontMetrics::width() is used and it is deprecated since Qt 13.0. + * In Qt 5.12 and before the QFontMetrics::width() is used and it is deprecated since Qt 5.13. * In Qt 5.11 the QFontMetrics::horizontalAdvance() was introduced. */ int TextWidth(const QFontMetrics& fm, const QString& text); @@ -303,6 +311,56 @@ namespace GUIUtil * Call QMenu::popup() only on supported QT_QPA_PLATFORM. */ void PopupMenu(QMenu* menu, const QPoint& point, QAction* at_action = nullptr); + + /** + * Returns the start-moment of the day in local time. + * + * QDateTime::QDateTime(const QDate& date) is deprecated since Qt 5.15. + * QDate::startOfDay() was introduced in Qt 5.14. + */ + QDateTime StartOfDay(const QDate& date); + + /** + * Returns true if pixmap has been set. + * + * QPixmap* QLabel::pixmap() is deprecated since Qt 5.15. + */ + bool HasPixmap(const QLabel* label); + QImage GetImage(const QLabel* label); + + /** + * Splits the string into substrings wherever separator occurs, and returns + * the list of those strings. Empty strings do not appear in the result. + * + * QString::split() signature differs in different Qt versions: + * - QString::SplitBehavior is deprecated since Qt 5.15 + * - Qt::SplitBehavior was introduced in Qt 5.14 + * If {QString|Qt}::SkipEmptyParts behavior is required, use this + * function instead of QString::split(). + */ + template <typename SeparatorType> + QStringList SplitSkipEmptyParts(const QString& string, const SeparatorType& separator) + { + #if (QT_VERSION >= QT_VERSION_CHECK(5, 14, 0)) + return string.split(separator, Qt::SkipEmptyParts); + #else + return string.split(separator, QString::SkipEmptyParts); + #endif + } + + /** + * Queue a function to run in an object's event loop. This can be + * replaced by a call to the QMetaObject::invokeMethod functor overload after Qt 5.10, but + * for now use a QObject::connect for compatibility with older Qt versions, based on + * https://stackoverflow.com/questions/21646467/how-to-execute-a-functor-or-a-lambda-in-a-given-thread-in-qt-gcd-style + */ + template <typename Fn> + void ObjectInvoke(QObject* object, Fn&& function, Qt::ConnectionType connection = Qt::QueuedConnection) + { + QObject source; + QObject::connect(&source, &QObject::destroyed, object, std::forward<Fn>(function), connection); + } + } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 7b07777641..1d8af5cbf6 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 4b91c19761..b097ef080c 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,12 +17,7 @@ #include <stdint.h> #include <QtDBus> #endif -// Include ApplicationServices.h after QtDbus to avoid redefinition of check(). -// This affects at least OSX 10.6. See /usr/include/AssertMacros.h for details. -// Note: This could also be worked around using: -// #define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 #ifdef Q_OS_MAC -#include <ApplicationServices/ApplicationServices.h> #include <qt/macnotificationhandler.h> #endif @@ -71,7 +66,7 @@ Notificator::~Notificator() #ifdef USE_DBUS -// Loosely based on http://www.qtcentre.org/archive/index.php/t-25879.html +// Loosely based on https://www.qtcentre.org/archive/index.php/t-25879.html class FreedesktopImage { public: diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index 667af2ec75..efe4b86f37 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index ae6aeb7709..416b9c83c9 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,6 +24,7 @@ #include <QIntValidator> #include <QLocale> #include <QMessageBox> +#include <QSettings> #include <QSystemTrayIcon> #include <QTimer> @@ -50,6 +51,13 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : #ifndef USE_UPNP ui->mapPortUpnp->setEnabled(false); #endif +#ifndef USE_NATPMP + ui->mapPortNatpmp->setEnabled(false); +#endif + connect(this, &QDialog::accepted, [this](){ + QSettings settings; + model->node().mapPort(settings.value("fUseUPnP").toBool(), settings.value("fUseNatpmp").toBool()); + }); ui->proxyIp->setEnabled(false); ui->proxyPort->setEnabled(false); @@ -130,8 +138,8 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : connect(ui->proxyPortTor, &QLineEdit::textChanged, this, &OptionsDialog::updateProxyValidationState); if (!QSystemTrayIcon::isSystemTrayAvailable()) { - ui->hideTrayIcon->setChecked(true); - ui->hideTrayIcon->setEnabled(false); + ui->showTrayIcon->setChecked(false); + ui->showTrayIcon->setEnabled(false); ui->minimizeToTray->setChecked(false); ui->minimizeToTray->setEnabled(false); } @@ -214,6 +222,7 @@ void OptionsDialog::setMapper() /* Network */ mapper->addMapping(ui->mapPortUpnp, OptionsModel::MapPortUPnP); + mapper->addMapping(ui->mapPortNatpmp, OptionsModel::MapPortNatpmp); mapper->addMapping(ui->allowIncoming, OptionsModel::Listen); mapper->addMapping(ui->connectSocks, OptionsModel::ProxyUse); @@ -227,7 +236,7 @@ void OptionsDialog::setMapper() /* Window */ #ifndef Q_OS_MAC if (QSystemTrayIcon::isSystemTrayAvailable()) { - mapper->addMapping(ui->hideTrayIcon, OptionsModel::HideTrayIcon); + mapper->addMapping(ui->showTrayIcon, OptionsModel::ShowTrayIcon); mapper->addMapping(ui->minimizeToTray, OptionsModel::MinimizeToTray); } mapper->addMapping(ui->minimizeOnClose, OptionsModel::MinimizeOnClose); @@ -286,17 +295,14 @@ void OptionsDialog::on_cancelButton_clicked() reject(); } -void OptionsDialog::on_hideTrayIcon_stateChanged(int fState) +void OptionsDialog::on_showTrayIcon_stateChanged(int state) { - if(fState) - { + if (state == Qt::Checked) { + ui->minimizeToTray->setEnabled(true); + } else { ui->minimizeToTray->setChecked(false); ui->minimizeToTray->setEnabled(false); } - else - { - ui->minimizeToTray->setEnabled(true); - } } void OptionsDialog::togglePruneWarning(bool enabled) diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index 568c8b6fd0..1cc96035c6 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -57,7 +57,7 @@ private Q_SLOTS: void on_okButton_clicked(); void on_cancelButton_clicked(); - void on_hideTrayIcon_stateChanged(int fState); + void on_showTrayIcon_stateChanged(int state); void togglePruneWarning(bool enabled); void showRestartWarning(bool fPersistent = false); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 7e089b4f95..1e0391a35c 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -13,11 +13,12 @@ #include <qt/guiutil.h> #include <interfaces/node.h> -#include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS +#include <mapport.h> #include <net.h> #include <netbase.h> -#include <txdb.h> // for -dbcache defaults +#include <txdb.h> // for -dbcache defaults #include <util/string.h> +#include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS #include <QDebug> #include <QSettings> @@ -54,14 +55,15 @@ void OptionsModel::Init(bool resetSettings) // These are Qt-only settings: // Window - if (!settings.contains("fHideTrayIcon")) + if (!settings.contains("fHideTrayIcon")) { settings.setValue("fHideTrayIcon", false); - fHideTrayIcon = settings.value("fHideTrayIcon").toBool(); - Q_EMIT hideTrayIconChanged(fHideTrayIcon); + } + m_show_tray_icon = !settings.value("fHideTrayIcon").toBool(); + Q_EMIT showTrayIconChanged(m_show_tray_icon); if (!settings.contains("fMinimizeToTray")) settings.setValue("fMinimizeToTray", false); - fMinimizeToTray = settings.value("fMinimizeToTray").toBool() && !fHideTrayIcon; + fMinimizeToTray = settings.value("fMinimizeToTray").toBool() && m_show_tray_icon; if (!settings.contains("fMinimizeOnClose")) settings.setValue("fMinimizeOnClose", false); @@ -122,6 +124,13 @@ void OptionsModel::Init(bool resetSettings) if (!gArgs.SoftSetBoolArg("-upnp", settings.value("fUseUPnP").toBool())) addOverriddenOption("-upnp"); + if (!settings.contains("fUseNatpmp")) { + settings.setValue("fUseNatpmp", DEFAULT_NATPMP); + } + if (!gArgs.SoftSetBoolArg("-natpmp", settings.value("fUseNatpmp").toBool())) { + addOverriddenOption("-natpmp"); + } + if (!settings.contains("fListen")) settings.setValue("fListen", DEFAULT_LISTEN); if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) @@ -219,7 +228,7 @@ static ProxySetting GetProxySetting(QSettings &settings, const QString &name) return default_val; } // contains IP at index 0 and port at index 1 - QStringList ip_port = settings.value(name).toString().split(":", QString::SkipEmptyParts); + QStringList ip_port = GUIUtil::SplitSkipEmptyParts(settings.value(name).toString(), ":"); if (ip_port.size() == 2) { return {true, ip_port.at(0), ip_port.at(1)}; } else { // Invalid: return default @@ -272,8 +281,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const { case StartAtStartup: return GUIUtil::GetStartOnSystemStartup(); - case HideTrayIcon: - return fHideTrayIcon; + case ShowTrayIcon: + return m_show_tray_icon; case MinimizeToTray: return fMinimizeToTray; case MapPortUPnP: @@ -281,7 +290,13 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return settings.value("fUseUPnP"); #else return false; -#endif +#endif // USE_UPNP + case MapPortNatpmp: +#ifdef USE_NATPMP + return settings.value("fUseNatpmp"); +#else + return false; +#endif // USE_NATPMP case MinimizeOnClose: return fMinimizeOnClose; @@ -342,10 +357,10 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in case StartAtStartup: successful = GUIUtil::SetStartOnSystemStartup(value.toBool()); break; - case HideTrayIcon: - fHideTrayIcon = value.toBool(); - settings.setValue("fHideTrayIcon", fHideTrayIcon); - Q_EMIT hideTrayIconChanged(fHideTrayIcon); + case ShowTrayIcon: + m_show_tray_icon = value.toBool(); + settings.setValue("fHideTrayIcon", !m_show_tray_icon); + Q_EMIT showTrayIconChanged(m_show_tray_icon); break; case MinimizeToTray: fMinimizeToTray = value.toBool(); @@ -353,7 +368,9 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in break; case MapPortUPnP: // core option - can be changed on-the-fly settings.setValue("fUseUPnP", value.toBool()); - node().mapPort(value.toBool()); + break; + case MapPortNatpmp: // core option - can be changed on-the-fly + settings.setValue("fUseNatpmp", value.toBool()); break; case MinimizeOnClose: fMinimizeOnClose = value.toBool(); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 3d9e7bbb80..f7171951a1 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -45,9 +45,10 @@ public: enum OptionID { StartAtStartup, // bool - HideTrayIcon, // bool + ShowTrayIcon, // bool MinimizeToTray, // bool MapPortUPnP, // bool + MapPortNatpmp, // bool MinimizeOnClose, // bool ProxyUse, // bool ProxyIP, // QString @@ -78,7 +79,7 @@ public: void setDisplayUnit(const QVariant &value); /* Explicit getters */ - bool getHideTrayIcon() const { return fHideTrayIcon; } + bool getShowTrayIcon() const { return m_show_tray_icon; } bool getMinimizeToTray() const { return fMinimizeToTray; } bool getMinimizeOnClose() const { return fMinimizeOnClose; } int getDisplayUnit() const { return nDisplayUnit; } @@ -100,7 +101,7 @@ public: private: interfaces::Node* m_node = nullptr; /* Qt-only settings */ - bool fHideTrayIcon; + bool m_show_tray_icon; bool fMinimizeToTray; bool fMinimizeOnClose; QString language; @@ -118,7 +119,7 @@ private: Q_SIGNALS: void displayUnitChanged(int unit); void coinControlFeaturesChanged(bool); - void hideTrayIconChanged(bool); + void showTrayIconChanged(bool); }; #endif // BITCOIN_QT_OPTIONSMODEL_H diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index b536567c8b..cca4dce624 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,6 +17,7 @@ #include <QAbstractItemDelegate> #include <QApplication> +#include <QDateTime> #include <QPainter> #include <QStatusTipEvent> diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index 4cf673b6a6..578ef601fb 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 6c2db52f63..96f6202874 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,7 +26,6 @@ #include <QApplication> #include <QByteArray> #include <QDataStream> -#include <QDateTime> #include <QDebug> #include <QFile> #include <QFileOpenEvent> diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index eaf2bafe59..08b83244ab 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 5220f8e138..bad81d894c 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -29,14 +29,16 @@ bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombine return pLeft->nodeid < pRight->nodeid; case PeerTableModel::Address: return pLeft->addrName.compare(pRight->addrName) < 0; - case PeerTableModel::Subversion: - return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; + case PeerTableModel::Network: + return pLeft->m_network < pRight->m_network; case PeerTableModel::Ping: return pLeft->m_min_ping_usec < pRight->m_min_ping_usec; case PeerTableModel::Sent: return pLeft->nSendBytes < pRight->nSendBytes; case PeerTableModel::Received: return pLeft->nRecvBytes < pRight->nRecvBytes; + case PeerTableModel::Subversion: + return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; } return false; @@ -104,7 +106,6 @@ PeerTableModel::PeerTableModel(interfaces::Node& node, QObject* parent) : m_node(node), timer(nullptr) { - columns << tr("NodeId") << tr("Node/Service") << tr("Ping") << tr("Sent") << tr("Received") << tr("User Agent"); priv.reset(new PeerTablePriv()); // set up timer for auto refresh @@ -133,13 +134,17 @@ void PeerTableModel::stopAutoRefresh() int PeerTableModel::rowCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return priv->size(); } int PeerTableModel::columnCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return columns.length(); } @@ -158,17 +163,21 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const case Address: // prepend to peer address down-arrow symbol for inbound connection and up-arrow for outbound connection return QString(rec->nodeStats.fInbound ? "↓ " : "↑ ") + QString::fromStdString(rec->nodeStats.addrName); - case Subversion: - return QString::fromStdString(rec->nodeStats.cleanSubVer); + case Network: + return GUIUtil::NetworkToQString(rec->nodeStats.m_network); case Ping: return GUIUtil::formatPingTime(rec->nodeStats.m_min_ping_usec); case Sent: return GUIUtil::formatBytes(rec->nodeStats.nSendBytes); case Received: return GUIUtil::formatBytes(rec->nodeStats.nRecvBytes); + case Subversion: + return QString::fromStdString(rec->nodeStats.cleanSubVer); } } else if (role == Qt::TextAlignmentRole) { switch (index.column()) { + case Network: + return QVariant(Qt::AlignCenter); case Ping: case Sent: case Received: @@ -176,6 +185,11 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const default: return QVariant(); } + } else if (role == StatsRole) { + switch (index.column()) { + case NetNodeId: return QVariant::fromValue(rec); + default: return QVariant(); + } } return QVariant(); @@ -211,11 +225,6 @@ QModelIndex PeerTableModel::index(int row, int column, const QModelIndex &parent return QModelIndex(); } -const CNodeCombinedStats *PeerTableModel::getNodeStats(int idx) -{ - return priv->index(idx); -} - void PeerTableModel::refresh() { Q_EMIT layoutAboutToBeChanged(); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 99de772ac0..7bff239507 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -28,6 +28,7 @@ struct CNodeCombinedStats { CNodeStateStats nodeStateStats; bool fNodeStateStatsAvailable; }; +Q_DECLARE_METATYPE(CNodeCombinedStats*) class NodeLessThan { @@ -52,7 +53,6 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); - const CNodeCombinedStats *getNodeStats(int idx); int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); void stopAutoRefresh(); @@ -60,10 +60,15 @@ public: enum ColumnIndex { NetNodeId = 0, Address = 1, - Ping = 2, - Sent = 3, - Received = 4, - Subversion = 5 + Network = 2, + Ping = 3, + Sent = 4, + Received = 5, + Subversion = 6 + }; + + enum { + StatsRole = Qt::UserRole, }; /** @name Methods overridden from QAbstractTableModel @@ -82,7 +87,7 @@ public Q_SLOTS: private: interfaces::Node& m_node; - QStringList columns; + const QStringList columns{tr("Peer Id"), tr("Address"), tr("Network"), tr("Ping"), tr("Sent"), tr("Received"), tr("User Agent")}; std::unique_ptr<PeerTablePriv> priv; QTimer *timer; }; diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index 52f1e60957..490826cbbb 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -98,15 +98,12 @@ bool QRImageWidget::setQR(const QString& data, const QString& text) QImage QRImageWidget::exportImage() { - if(!pixmap()) - return QImage(); - return pixmap()->toImage(); + return GUIUtil::GetImage(this); } void QRImageWidget::mousePressEvent(QMouseEvent *event) { - if(event->button() == Qt::LeftButton && pixmap()) - { + if (event->button() == Qt::LeftButton && GUIUtil::HasPixmap(this)) { event->accept(); QMimeData *mimeData = new QMimeData; mimeData->setImageData(exportImage()); @@ -121,7 +118,7 @@ void QRImageWidget::mousePressEvent(QMouseEvent *event) void QRImageWidget::saveImage() { - if(!pixmap()) + if (!GUIUtil::HasPixmap(this)) return; QString fn = GUIUtil::getSaveFileName(this, tr("Save QR Code"), QString(), tr("PNG Image (*.png)"), nullptr); if (!fn.isEmpty()) @@ -132,14 +129,14 @@ void QRImageWidget::saveImage() void QRImageWidget::copyImage() { - if(!pixmap()) + if (!GUIUtil::HasPixmap(this)) return; QApplication::clipboard()->setImage(exportImage()); } void QRImageWidget::contextMenuEvent(QContextMenuEvent *event) { - if(!pixmap()) + if (!GUIUtil::HasPixmap(this)) return; contextMenu->exec(event->globalPos()); } diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index a031bd7632..5dbba074a1 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvalidatedlineedit.h b/src/qt/qvalidatedlineedit.h index 2c72b2ecda..b32305f5e1 100644 --- a/src/qt/qvalidatedlineedit.h +++ b/src/qt/qvalidatedlineedit.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index d374d610ee..1a9cd78d1e 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 27455e2906..9b89bd6a8b 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index d385c42821..bef3edc5fe 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 846478643d..c861680761 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 3e20368a36..18b913774b 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,15 +36,17 @@ RecentRequestsTableModel::~RecentRequestsTableModel() int RecentRequestsTableModel::rowCount(const QModelIndex &parent) const { - Q_UNUSED(parent); - + if (parent.isValid()) { + return 0; + } return list.length(); } int RecentRequestsTableModel::columnCount(const QModelIndex &parent) const { - Q_UNUSED(parent); - + if (parent.isValid()) { + return 0; + } return columns.length(); } diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index c0bd3461bb..b817b64e77 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/res/animation/makespinner.sh b/src/qt/res/animation/makespinner.sh index 4fa8dadf86..83142f5034 100755 --- a/src/qt/res/animation/makespinner.sh +++ b/src/qt/res/animation/makespinner.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/res/bitcoin-qt-res.rc b/src/qt/res/bitcoin-qt-res.rc index 94ae256477..e590b407b0 100644 --- a/src/qt/res/bitcoin-qt-res.rc +++ b/src/qt/res/bitcoin-qt-res.rc @@ -4,8 +4,8 @@ IDI_ICON2 ICON DISCARDABLE "icons/bitcoin_testnet.ico" #include <windows.h> // needed for VERSIONINFO #include "../../clientversion.h" // holds the needed client version information -#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_REVISION,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) +#define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD +#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION #define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 4c5601242e..df98dbbc99 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -25,10 +25,14 @@ #include <univalue.h> #ifdef ENABLE_WALLET +#ifdef USE_BDB +#include <wallet/bdb.h> +#endif #include <wallet/db.h> #include <wallet/wallet.h> #endif +#include <QDateTime> #include <QFont> #include <QKeyEvent> #include <QMenu> @@ -453,10 +457,13 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty move(QGuiApplication::primaryScreen()->availableGeometry().center() - frameGeometry().center()); } + ui->splitter->restoreState(settings.value("PeersTabSplitterSizes").toByteArray()); + QChar nonbreaking_hyphen(8209); ui->dataDir->setToolTip(ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg(QString(nonbreaking_hyphen) + "blocksdir")); ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); + ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg("addnode").arg(QString(nonbreaking_hyphen) + "addnode").arg(QString(nonbreaking_hyphen) + "connect")); if (platformStyle->getImagesOnButtons()) { ui->openDebugLogfileButton->setIcon(platformStyle->SingleColorIcon(":/icons/export")); @@ -479,13 +486,6 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->WalletSelector->setVisible(false); ui->WalletSelectorLabel->setVisible(false); - // set library version labels -#ifdef ENABLE_WALLET - ui->berkeleyDBVersion->setText(QString::fromStdString(BerkeleyDatabaseVersion())); -#else - ui->label_berkeleyDBVersion->hide(); - ui->berkeleyDBVersion->hide(); -#endif // Register RPC timer interface rpcTimerInterface = new QtRPCTimerInterface(); // avoid accidentally overwriting an existing, non QTThread @@ -493,9 +493,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty m_node.rpcSetTimerInterfaceIfUnset(rpcTimerInterface); setTrafficGraphRange(INITIAL_TRAFFIC_GRAPH_MINS); - - ui->detailWidget->hide(); - ui->peerHeading->setText(tr("Select a peer to view detailed information.")); + updateDetailWidget(); consoleFontSize = settings.value(fontSizeSettingsKey, QFont().pointSize()).toInt(); clear(); @@ -507,6 +505,7 @@ RPCConsole::~RPCConsole() { QSettings settings; settings.setValue("RPCConsoleWindowGeometry", saveGeometry()); + settings.setValue("PeersTabSplitterSizes", ui->splitter->saveState()); m_node.rpcUnsetTimerInterface(rpcTimerInterface); delete rpcTimerInterface; delete ui; @@ -626,7 +625,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ connect(disconnectAction, &QAction::triggered, this, &RPCConsole::disconnectSelectedNode); // peer table signal handling - update peer details when selecting new node - connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::peerSelected); + connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); // peer table signal handling - update peer details when new nodes are added to the model connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::peerLayoutChanged); // peer table signal handling - cache selected node ids @@ -1021,25 +1020,11 @@ void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } -void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelection &deselected) -{ - Q_UNUSED(deselected); - - if (!clientModel || !clientModel->getPeerTableModel() || selected.indexes().isEmpty()) - return; - - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected.indexes().first().row()); - if (stats) - updateNodeDetail(stats); -} - void RPCConsole::peerLayoutAboutToChange() { - QModelIndexList selected = ui->peerWidget->selectionModel()->selectedIndexes(); cachedNodeids.clear(); - for(int i = 0; i < selected.size(); i++) - { - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected.at(i).row()); + for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { + const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); cachedNodeids.append(stats->nodeStats.nodeid); } } @@ -1049,7 +1034,6 @@ void RPCConsole::peerLayoutChanged() if (!clientModel || !clientModel->getPeerTableModel()) return; - const CNodeCombinedStats *stats = nullptr; bool fUnselect = false; bool fReselect = false; @@ -1080,9 +1064,6 @@ void RPCConsole::peerLayoutChanged() fUnselect = true; fReselect = true; } - - // get fresh stats on the detail node. - stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); } if (fUnselect && selectedRow >= 0) { @@ -1097,15 +1078,21 @@ void RPCConsole::peerLayoutChanged() } } - if (stats) - updateNodeDetail(stats); + updateDetailWidget(); } -void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) +void RPCConsole::updateDetailWidget() { + const QList<QModelIndex> selected_peers = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); + if (!clientModel || !clientModel->getPeerTableModel() || selected_peers.size() != 1) { + ui->detailWidget->hide(); + ui->peerHeading->setText(tr("Select a peer to view detailed information.")); + return; + } + const auto stats = selected_peers.first().data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); // update the detail ui with latest node information QString peerAddrDetails(QString::fromStdString(stats->nodeStats.addrName) + " "); - peerAddrDetails += tr("(node id: %1)").arg(QString::number(stats->nodeStats.nodeid)); + peerAddrDetails += tr("(peer id: %1)").arg(QString::number(stats->nodeStats.nodeid)); if (!stats->nodeStats.addrLocal.empty()) peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); @@ -1121,8 +1108,8 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); - ui->peerHeight->setText(QString::number(stats->nodeStats.nStartingHeight)); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type)); + ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == PF_NONE) { ui->peerPermissions->setText(tr("N/A")); } else { @@ -1148,6 +1135,8 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerCommonHeight->setText(QString("%1").arg(stats->nodeStateStats.nCommonHeight)); else ui->peerCommonHeight->setText(tr("Unknown")); + + ui->peerHeight->setText(QString::number(stats->nodeStateStats.m_starting_height)); } ui->detailWidget->show(); @@ -1213,19 +1202,9 @@ void RPCConsole::banSelectedNode(int bantime) if (!clientModel) return; - // Get selected peer addresses - QList<QModelIndex> nodes = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); - for(int i = 0; i < nodes.count(); i++) - { - // Get currently selected peer address - NodeId id = nodes.at(i).data().toLongLong(); - - // Get currently selected peer address - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); - if (detailNodeRow < 0) return; - + for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { // Find possible nodes, ban it and clear the selected node - const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(detailNodeRow); + const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); if (stats) { m_node.ban(stats->nodeStats.addr, bantime); m_node.disconnectByAddress(stats->nodeStats.addr); @@ -1260,8 +1239,7 @@ void RPCConsole::clearSelectedNode() { ui->peerWidget->selectionModel()->clearSelection(); cachedNodeids.clear(); - ui->detailWidget->hide(); - ui->peerHeading->setText(tr("Select a peer to view detailed information.")); + updateDetailWidget(); } void RPCConsole::showOrHideBanTableIfRequired() diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 280c5bd71a..5f308dc36d 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,6 +28,7 @@ namespace Ui { } QT_BEGIN_NAMESPACE +class QDateTime; class QMenu; class QItemSelection; QT_END_NAMESPACE @@ -94,6 +95,8 @@ private Q_SLOTS: void showOrHideBanTableIfRequired(); /** clear the selected node */ void clearSelectedNode(); + /** show detailed information on ui about selected node */ + void updateDetailWidget(); public Q_SLOTS: void clear(bool clearHistory = true); @@ -115,8 +118,6 @@ public Q_SLOTS: void browseHistory(int offset); /** Scroll console view to end */ void scrollToEnd(); - /** Handle selection of peer in peers list */ - void peerSelected(const QItemSelection &selected, const QItemSelection &deselected); /** Handle selection caching before update */ void peerLayoutAboutToChange(); /** Handle updated peer information */ @@ -137,8 +138,6 @@ Q_SIGNALS: private: void startExecutor(); void setTrafficGraphRange(int mins); - /** show detailed information on ui about selected node */ - void updateNodeDetail(const CNodeCombinedStats *stats); enum ColumnWidths { diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 63b06b6215..9eedd8cf75 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -35,7 +35,7 @@ #include <QSettings> #include <QTextDocument> -static const std::array<int, 9> confTargets = { {2, 4, 6, 12, 24, 48, 144, 504, 1008} }; +static constexpr std::array confTargets{2, 4, 6, 12, 24, 48, 144, 504, 1008}; int getConfTargetForIndex(int index) { if (index+1 > static_cast<int>(confTargets.size())) { return confTargets.back(); @@ -173,8 +173,15 @@ void SendCoinsDialog::setModel(WalletModel *_model) } connect(ui->confTargetSelector, static_cast<void (QComboBox::*)(int)>(&QComboBox::currentIndexChanged), this, &SendCoinsDialog::updateSmartFeeLabel); connect(ui->confTargetSelector, static_cast<void (QComboBox::*)(int)>(&QComboBox::currentIndexChanged), this, &SendCoinsDialog::coinControlUpdateLabels); + +#if (QT_VERSION >= QT_VERSION_CHECK(5, 15, 0)) + connect(ui->groupFee, &QButtonGroup::idClicked, this, &SendCoinsDialog::updateFeeSectionControls); + connect(ui->groupFee, &QButtonGroup::idClicked, this, &SendCoinsDialog::coinControlUpdateLabels); +#else connect(ui->groupFee, static_cast<void (QButtonGroup::*)(int)>(&QButtonGroup::buttonClicked), this, &SendCoinsDialog::updateFeeSectionControls); connect(ui->groupFee, static_cast<void (QButtonGroup::*)(int)>(&QButtonGroup::buttonClicked), this, &SendCoinsDialog::coinControlUpdateLabels); +#endif + connect(ui->customFee, &BitcoinAmountField::valueChanged, this, &SendCoinsDialog::coinControlUpdateLabels); connect(ui->optInRBF, &QCheckBox::stateChanged, this, &SendCoinsDialog::updateSmartFeeLabel); connect(ui->optInRBF, &QCheckBox::stateChanged, this, &SendCoinsDialog::coinControlUpdateLabels); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 8863d2e5c8..4fc2f57cd6 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 6619faf417..01135cdfef 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4835dd7954..6c110f0688 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -120,7 +120,7 @@ void SignVerifyMessageDialog::on_signMessageButton_SM_clicked() ui->statusLabel_SM->setText(tr("The entered address is invalid.") + QString(" ") + tr("Please check the address and try again.")); return; } - const PKHash* pkhash = boost::get<PKHash>(&destination); + const PKHash* pkhash = std::get_if<PKHash>(&destination); if (!pkhash) { ui->addressIn_SM->setValid(false); ui->statusLabel_SM->setStyleSheet("QLabel { color: red; }"); diff --git a/src/qt/signverifymessagedialog.h b/src/qt/signverifymessagedialog.h index d33a2d038d..d98cb290a1 100644 --- a/src/qt/signverifymessagedialog.h +++ b/src/qt/signverifymessagedialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index f00f086d1e..5796a6ef56 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -25,8 +25,8 @@ #include <QScreen> -SplashScreen::SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle) : - QWidget(nullptr, f), curAlignment(0) +SplashScreen::SplashScreen(const NetworkStyle* networkStyle) + : QWidget(), curAlignment(0) { // set reference point, paddings int paddingRight = 50; diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index a0cd677d3d..386039291c 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,7 +28,7 @@ class SplashScreen : public QWidget Q_OBJECT public: - explicit SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle); + explicit SplashScreen(const NetworkStyle *networkStyle); ~SplashScreen(); void setNode(interfaces::Node& node); diff --git a/src/qt/test/addressbooktests.h b/src/qt/test/addressbooktests.h index 5de89c7592..ab52206466 100644 --- a/src/qt/test/addressbooktests.h +++ b/src/qt/test/addressbooktests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ class Node; class AddressBookTests : public QObject { public: - AddressBookTests(interfaces::Node& node) : m_node(node) {} + explicit AddressBookTests(interfaces::Node& node) : m_node(node) {} interfaces::Node& m_node; Q_OBJECT diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index de1fbcb94c..24d7735447 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,17 +14,26 @@ #include <QDir> #include <QtGlobal> -static UniValue rpcNestedTest_rpc(const JSONRPCRequest& request) +static RPCHelpMan rpcNestedTest_rpc() { - if (request.fHelp) { - return "help message"; - } - return request.params.write(0, 0); + return RPCHelpMan{ + "rpcNestedTest", + "echo the passed string(s)", + { + {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, + {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, + {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, + }, + {}, + RPCExamples{""}, + [](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + return request.params.write(0, 0); + }, + }; } -static const CRPCCommand vRPCCommands[] = -{ - { "test", "rpcNestedTest", &rpcNestedTest_rpc, {} }, +static const CRPCCommand vRPCCommands[] = { + {"test", "rpcNestedTest", &rpcNestedTest_rpc, {"arg1", "arg2", "arg3"}}, }; void RPCNestedTests::rpcNestedTests() @@ -34,41 +43,41 @@ void RPCNestedTests::rpcNestedTests() tableRPC.appendCommand("rpcNestedTest", &vRPCCommands[0]); TestingSetup test; + m_node.setContext(&test.m_node); if (RPCIsInWarmup(nullptr)) SetRPCWarmupFinished(); std::string result; std::string result2; std::string filtered; - interfaces::Node* node = &m_node; - RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()[chain]", &filtered); //simple result filtering with path + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()[chain]", &filtered); //simple result filtering with path QVERIFY(result=="main"); QVERIFY(filtered == "getblockchaininfo()[chain]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "getblock(getbestblockhash())"); //simple 2 level nesting - RPCConsole::RPCExecuteCommandLine(*node, result, "getblock(getblock(getbestblockhash())[hash], true)"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblock(getbestblockhash())"); //simple 2 level nesting + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblock(getblock(getbestblockhash())[hash], true)"); - RPCConsole::RPCExecuteCommandLine(*node, result, "getblock( getblock( getblock(getbestblockhash())[hash] )[hash], true)"); //4 level nesting with whitespace, filtering path and boolean parameter + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblock( getblock( getblock(getbestblockhash())[hash] )[hash], true)"); //4 level nesting with whitespace, filtering path and boolean parameter - RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo"); QVERIFY(result.substr(0,1) == "{"); - RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()"); QVERIFY(result.substr(0,1) == "{"); - RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo "); //whitespace at the end will be tolerated + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo "); //whitespace at the end will be tolerated QVERIFY(result.substr(0,1) == "{"); - (RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()[\"chain\"]")); //Quote path identifier are allowed, but look after a child containing the quotes in the key + (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()[\"chain\"]")); //Quote path identifier are allowed, but look after a child containing the quotes in the key QVERIFY(result == "null"); - (RPCConsole::RPCExecuteCommandLine(*node, result, "createrawtransaction [] {} 0")); //parameter not in brackets are allowed - (RPCConsole::RPCExecuteCommandLine(*node, result2, "createrawtransaction([],{},0)")); //parameter in brackets are allowed + (RPCConsole::RPCExecuteCommandLine(m_node, result, "createrawtransaction [] {} 0")); //parameter not in brackets are allowed + (RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction([],{},0)")); //parameter in brackets are allowed QVERIFY(result == result2); - (RPCConsole::RPCExecuteCommandLine(*node, result2, "createrawtransaction( [], {} , 0 )")); //whitespace between parameters is allowed + (RPCConsole::RPCExecuteCommandLine(m_node, result2, "createrawtransaction( [], {} , 0 )")); //whitespace between parameters is allowed QVERIFY(result == result2); - RPCConsole::RPCExecuteCommandLine(*node, result, "getblock(getbestblockhash())[tx][0]", &filtered); + RPCConsole::RPCExecuteCommandLine(m_node, result, "getblock(getbestblockhash())[tx][0]", &filtered); QVERIFY(result == "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"); QVERIFY(filtered == "getblock(getbestblockhash())[tx][0]"); @@ -93,35 +102,35 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCParseCommandLine(nullptr, result, "help(importprivkey(abc), walletpassphrase(def))", false, &filtered); QVERIFY(filtered == "help(importprivkey(…), walletpassphrase(…))"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest"); QVERIFY(result == "[]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest ''"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest ''"); QVERIFY(result == "[\"\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest \"\""); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest \"\""); QVERIFY(result == "[\"\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest '' abc"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest '' abc"); QVERIFY(result == "[\"\",\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc '' abc"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest abc '' abc"); QVERIFY(result == "[\"abc\",\"\",\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc abc"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest abc abc"); QVERIFY(result == "[\"abc\",\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc\t\tabc"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest abc\t\tabc"); QVERIFY(result == "[\"abc\",\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc )"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest(abc )"); QVERIFY(result == "[\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest( abc )"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest( abc )"); QVERIFY(result == "[\"abc\"]"); - RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest( abc , cba )"); + RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax - (RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo(")); //tolerate non closing brackets if we have no arguments - (RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()()()")); //tolerate non command brackts - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo(True)"), UniValue); //invalid argument - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "a(getblockchaininfo(True))"), UniValue); //method not found - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tollerate empty arguments when using , - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,abc)"), std::runtime_error); //don't tollerate empty arguments when using , - QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax + (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo(")); //tolerate non closing brackets if we have no arguments + (RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo()()()")); //tolerate non command brackets + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "getblockchaininfo(True)"), UniValue); //invalid argument + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "a(getblockchaininfo(True))"), UniValue); //method not found + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tolerate empty arguments when using , + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest(abc,,abc)"), std::runtime_error); //don't tolerate empty arguments when using , + QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(m_node, result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tolerate empty arguments when using , } diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 0a00d1113a..28dc7140e1 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ class Node; class RPCNestedTests : public QObject { public: - RPCNestedTests(interfaces::Node& node) : m_node(node) {} + explicit RPCNestedTests(interfaces::Node& node) : m_node(node) {} interfaces::Node& m_node; Q_OBJECT diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 86356b43c8..9ef5fe8fc7 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index 759832381b..987d921f03 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 950314da3b..df5931a032 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/wallettests.h b/src/qt/test/wallettests.h index 8ee40bf07f..6044bedb1d 100644 --- a/src/qt/test/wallettests.h +++ b/src/qt/test/wallettests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ class Node; class WalletTests : public QObject { public: - WalletTests(interfaces::Node& node) : m_node(node) {} + explicit WalletTests(interfaces::Node& node) : m_node(node) {} interfaces::Node& m_node; Q_OBJECT diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 6428fc4daf..dabdf9d887 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index 715e312b19..b547bf7645 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index d6bb84f7e6..693b363692 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 632a18de5c..77fec93f0f 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -123,7 +123,7 @@ QList<TransactionRecord> TransactionRecord::decomposeTransaction(const interface continue; } - if (!boost::get<CNoDestination>(&wtx.txout_address[nOut])) + if (!std::get_if<CNoDestination>(&wtx.txout_address[nOut])) { // Sent to Bitcoin Address sub.type = TransactionRecord::SendToAddress; diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index c983c527c0..e10243a28a 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 3148089b52..05b250eea9 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -289,13 +289,17 @@ void TransactionTableModel::updateConfirmations() int TransactionTableModel::rowCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return priv->size(); } int TransactionTableModel::columnCount(const QModelIndex &parent) const { - Q_UNUSED(parent); + if (parent.isValid()) { + return 0; + } return columns.length(); } diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index 4b699d4d7d..f8576edd59 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 54ecfc38ec..0cf6480b82 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -237,7 +237,7 @@ void TransactionView::setModel(WalletModel *_model) if (_model->getOptionsModel()) { // Add third party transaction URLs to context menu - QStringList listUrls = _model->getOptionsModel()->getThirdPartyTxUrls().split("|", QString::SkipEmptyParts); + QStringList listUrls = GUIUtil::SplitSkipEmptyParts(_model->getOptionsModel()->getThirdPartyTxUrls(), "|"); for (int i = 0; i < listUrls.size(); ++i) { QString url = listUrls[i].trimmed(); @@ -275,30 +275,30 @@ void TransactionView::chooseDate(int idx) break; case Today: transactionProxyModel->setDateRange( - QDateTime(current), + GUIUtil::StartOfDay(current), TransactionFilterProxy::MAX_DATE); break; case ThisWeek: { // Find last Monday QDate startOfWeek = current.addDays(-(current.dayOfWeek()-1)); transactionProxyModel->setDateRange( - QDateTime(startOfWeek), + GUIUtil::StartOfDay(startOfWeek), TransactionFilterProxy::MAX_DATE); } break; case ThisMonth: transactionProxyModel->setDateRange( - QDateTime(QDate(current.year(), current.month(), 1)), + GUIUtil::StartOfDay(QDate(current.year(), current.month(), 1)), TransactionFilterProxy::MAX_DATE); break; case LastMonth: transactionProxyModel->setDateRange( - QDateTime(QDate(current.year(), current.month(), 1).addMonths(-1)), - QDateTime(QDate(current.year(), current.month(), 1))); + GUIUtil::StartOfDay(QDate(current.year(), current.month(), 1).addMonths(-1)), + GUIUtil::StartOfDay(QDate(current.year(), current.month(), 1))); break; case ThisYear: transactionProxyModel->setDateRange( - QDateTime(QDate(current.year(), 1, 1)), + GUIUtil::StartOfDay(QDate(current.year(), 1, 1)), TransactionFilterProxy::MAX_DATE); break; case Range: @@ -583,8 +583,8 @@ void TransactionView::dateRangeChanged() if(!transactionProxyModel) return; transactionProxyModel->setDateRange( - QDateTime(dateFrom->date()), - QDateTime(dateTo->date()).addDays(1)); + GUIUtil::StartOfDay(dateFrom->date()), + GUIUtil::StartOfDay(dateTo->date()).addDays(1)); } void TransactionView::focusTransaction(const QModelIndex &idx) diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 9ce7f4ad97..b268823066 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d9e0274d01..7a89325ddf 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -128,10 +128,20 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal } // Instantiate model and register it. - WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, nullptr); - // Handler callback runs in a different thread so fix wallet model thread affinity. + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, + nullptr /* required for the following moveToThread() call */); + + // Move WalletModel object to the thread that created the WalletController + // object (GUI main thread), instead of the current thread, which could be + // an outside wallet thread or RPC thread sending a LoadWallet notification. + // This ensures queued signals sent to the WalletModel object will be + // handled on the GUI event loop. wallet_model->moveToThread(thread()); - wallet_model->setParent(this); + // setParent(parent) must be called in the thread which created the parent object. More details in #18948. + GUIUtil::ObjectInvoke(this, [wallet_model, this] { + wallet_model->setParent(this); + }, GUIUtil::blockingGUIThreadConnection()); + m_wallets.push_back(wallet_model); // WalletModel::startPollBalance needs to be called in a thread managed by @@ -142,7 +152,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal connect(wallet_model, &WalletModel::unload, this, [this, wallet_model] { // Defer removeAndDeleteWallet when no modal widget is active. - // TODO: remove this workaround by removing usage of QDiallog::exec. + // TODO: remove this workaround by removing usage of QDialog::exec. if (QApplication::activeModalWidget()) { connect(qApp, &QApplication::focusWindowChanged, wallet_model, [this, wallet_model]() { if (!QApplication::activeModalWidget()) { @@ -157,7 +167,6 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal // Re-emit coinsSent signal from wallet model. connect(wallet_model, &WalletModel::coinsSent, this, &WalletController::coinsSent); - // Notify walletAdded signal on the GUI thread. Q_EMIT walletAdded(wallet_model); return wallet_model; diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 4a9b4a5c84..2edb7eff8a 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -196,11 +196,11 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } } -void WalletFrame::encryptWallet(bool status) +void WalletFrame::encryptWallet() { WalletView *walletView = currentWalletView(); if (walletView) - walletView->encryptWallet(status); + walletView->encryptWallet(); } void WalletFrame::backupWallet() diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index e2fa8055bd..f57f8678d6 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -86,7 +86,7 @@ public Q_SLOTS: void gotoLoadPSBT(bool from_clipboard = false); /** Encrypt the wallet */ - void encryptWallet(bool status); + void encryptWallet(); /** Backup the wallet */ void backupWallet(); /** Change encrypted wallet passphrase */ diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 6a3f903206..cad472b43b 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -313,12 +313,9 @@ WalletModel::EncryptionStatus WalletModel::getEncryptionStatus() const } } -bool WalletModel::setWalletEncrypted(bool encrypted, const SecureString &passphrase) +bool WalletModel::setWalletEncrypted(const SecureString& passphrase) { - if (encrypted) { - return m_wallet->encryptWallet(passphrase); - } - return false; + return m_wallet->encryptWallet(passphrase); } bool WalletModel::setWalletLocked(bool locked, const SecureString &passPhrase) diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index fd52db2da3..9a3c3f2f66 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -105,7 +105,7 @@ public: SendCoinsReturn sendCoins(WalletModelTransaction &transaction); // Wallet encryption - bool setWalletEncrypted(bool encrypted, const SecureString &passphrase); + bool setWalletEncrypted(const SecureString& passphrase); // Passphrase only needed when unlocking bool setWalletLocked(bool locked, const SecureString &passPhrase=SecureString()); bool changePassphrase(const SecureString &oldPass, const SecureString &newPass); diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 2fc883a5f5..b1e6b43e60 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -258,11 +258,11 @@ void WalletView::updateEncryptionStatus() Q_EMIT encryptionStatusChanged(); } -void WalletView::encryptWallet(bool status) +void WalletView::encryptWallet() { if(!walletModel) return; - AskPassphraseDialog dlg(status ? AskPassphraseDialog::Encrypt : AskPassphraseDialog::Decrypt, this); + AskPassphraseDialog dlg(AskPassphraseDialog::Encrypt, this); dlg.setModel(walletModel); dlg.exec(); diff --git a/src/qt/walletview.h b/src/qt/walletview.h index f186554758..68f8a5e95b 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -92,7 +92,7 @@ public Q_SLOTS: */ void processNewTransaction(const QModelIndex& parent, int start, int /*end*/); /** Encrypt the wallet */ - void encryptWallet(bool status); + void encryptWallet(); /** Backup the wallet */ void backupWallet(); /** Change encrypted wallet passphrase */ diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 07122b7f6d..9248db1539 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -53,7 +53,7 @@ #include <sys/vmmeter.h> #endif #endif -#ifdef __linux__ +#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) #include <sys/auxv.h> #endif @@ -69,7 +69,7 @@ void RandAddSeedPerfmon(CSHA512& hasher) // This can take up to 2 seconds, so only do it every 10 minutes. // Initialize last_perfmon to 0 seconds, we don't skip the first call. - static std::atomic<std::chrono::seconds> last_perfmon{std::chrono::seconds{0}}; + static std::atomic<std::chrono::seconds> last_perfmon{0s}; auto last_time = last_perfmon.load(); auto current_time = GetTime<std::chrono::seconds>(); if (current_time < last_time + std::chrono::minutes{10}) return; @@ -326,7 +326,7 @@ void RandAddStaticEnv(CSHA512& hasher) // Bitcoin client version hasher << CLIENT_VERSION; -#ifdef __linux__ +#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) // Information available through getauxval() # ifdef AT_HWCAP hasher << getauxval(AT_HWCAP); @@ -346,7 +346,7 @@ void RandAddStaticEnv(CSHA512& hasher) const char* exec_str = (const char*)getauxval(AT_EXECFN); if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); # endif -#endif // __linux__ +#endif // HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL #ifdef HAVE_GETCPUID AddAllCPUID(hasher); diff --git a/src/rest.cpp b/src/rest.cpp index 949cc9d84a..678ffdb760 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 392073d047..3f97f6f3d9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -17,6 +17,7 @@ #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> +#include <policy/fees.h> #include <policy/feerate.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -65,7 +66,7 @@ NodeContext& EnsureNodeContext(const util::Ref& context) CTxMemPool& EnsureMemPool(const util::Ref& context) { - NodeContext& node = EnsureNodeContext(context); + const NodeContext& node = EnsureNodeContext(context); if (!node.mempool) { throw JSONRPCError(RPC_CLIENT_MEMPOOL_DISABLED, "Mempool disabled or instance not found"); } @@ -74,13 +75,22 @@ CTxMemPool& EnsureMemPool(const util::Ref& context) ChainstateManager& EnsureChainman(const util::Ref& context) { - NodeContext& node = EnsureNodeContext(context); + const NodeContext& node = EnsureNodeContext(context); if (!node.chainman) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Node chainman not found"); } return *node.chainman; } +CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context) +{ + NodeContext& node = EnsureNodeContext(context); + if (!node.fee_estimator) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Fee estimation disabled"); + } + return *node.fee_estimator; +} + /* Calculate the difficulty for a given block index. */ double GetDifficulty(const CBlockIndex* blockindex) @@ -162,16 +172,21 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn result.pushKV("versionHex", strprintf("%08x", block.nVersion)); result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); UniValue txs(UniValue::VARR); - for(const auto& tx : block.vtx) - { - if(txDetails) - { + if (txDetails) { + CBlockUndo blockUndo; + const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); + for (size_t i = 0; i < block.vtx.size(); ++i) { + const CTransactionRef& tx = block.vtx.at(i); + // coinbase transaction (i == 0) doesn't have undo data + const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags()); + TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); txs.push_back(objTx); } - else + } else { + for (const CTransactionRef& tx : block.vtx) { txs.push_back(tx->GetHash().GetHex()); + } } result.pushKV("tx", txs); result.pushKV("time", block.GetBlockTime()); @@ -926,6 +941,7 @@ static RPCHelpMan getblock() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::ELISION, "", "The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 \"tx\" result"}, + {RPCResult::Type::NUM, "fee", "The transaction fee in " + CURRENCY_UNIT + ", omitted if block undo data is not available"}, }}, }}, }}, @@ -1267,7 +1283,7 @@ RPCHelpMan getblockchaininfo() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "chain", "current network name (main, test, regtest)"}, + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, {RPCResult::Type::NUM, "blocks", "the height of the most-work fully-validated chain. The genesis block has height 0"}, {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 5b362bf211..e4ce80400e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; +class CBlockPolicyEstimator; class CTxMemPool; class ChainstateManager; class UniValue; @@ -54,5 +55,6 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], NodeContext& EnsureNodeContext(const util::Ref& context); CTxMemPool& EnsureMemPool(const util::Ref& context); ChainstateManager& EnsureChainman(const util::Ref& context); +CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context); #endif diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 6522c0d73e..0e96beb6db 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -242,12 +242,7 @@ static RPCHelpMan generatetodescriptor() static RPCHelpMan generate() { return RPCHelpMan{"generate", "has been replaced by the -generate cli option. Refer to -help for more information.", {}, {}, RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - - if (request.fHelp) { - throw std::runtime_error(self.ToString()); - } else { throw JSONRPCError(RPC_METHOD_NOT_FOUND, self.ToString()); - } }}; } @@ -414,7 +409,7 @@ static RPCHelpMan getmininginfo() {RPCResult::Type::NUM, "difficulty", "The current difficulty"}, {RPCResult::Type::NUM, "networkhashps", "The network hashes per second"}, {RPCResult::Type::NUM, "pooledtx", "The size of the mempool"}, - {RPCResult::Type::STR, "chain", "current network name (main, test, regtest)"}, + {RPCResult::Type::STR, "chain", "current network name (main, test, signet, regtest)"}, {RPCResult::Type::STR, "warnings", "any network and blockchain warnings"}, }}, RPCExamples{ @@ -663,11 +658,15 @@ static RPCHelpMan getblocktemplate() if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + if (!Params().IsTestChain()) { + if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + } - if (::ChainstateActive().IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + if (::ChainstateActive().IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + } + } static unsigned int nTransactionsUpdatedLast; const CTxMemPool& mempool = EnsureMemPool(request.context); @@ -719,6 +718,13 @@ static RPCHelpMan getblocktemplate() // TODO: Maybe recheck connections/IBD and (if something wrong) send an expires-immediately template to stop miners? } + const Consensus::Params& consensusParams = Params().GetConsensus(); + + // GBT must be called with 'signet' set in the rules for signet chains + if (consensusParams.signet_blocks && setClientRules.count("signet") != 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "getblocktemplate must be called with the signet rule set (call with {\"rules\": [\"segwit\", \"signet\"]})"); + } + // GBT must be called with 'segwit' set in the rules if (setClientRules.count("segwit") != 1) { throw JSONRPCError(RPC_INVALID_PARAMETER, "getblocktemplate must be called with the segwit rule set (call with {\"rules\": [\"segwit\"]})"); @@ -750,7 +756,6 @@ static RPCHelpMan getblocktemplate() } CHECK_NONFATAL(pindexPrev); CBlock* pblock = &pblocktemplate->block; // pointer for convenience - const Consensus::Params& consensusParams = Params().GetConsensus(); // Update nTime UpdateTime(pblock, consensusParams, pindexPrev); @@ -814,6 +819,12 @@ static RPCHelpMan getblocktemplate() UniValue aRules(UniValue::VARR); aRules.push_back("csv"); if (!fPreSegWit) aRules.push_back("!segwit"); + if (consensusParams.signet_blocks) { + // indicate to miner that they must understand signet rules + // when attempting to mine with this template + aRules.push_back("!signet"); + } + UniValue vbavailable(UniValue::VOBJ); for (int j = 0; j < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { Consensus::DeploymentPos pos = Consensus::DeploymentPos(j); @@ -894,6 +905,10 @@ static RPCHelpMan getblocktemplate() result.pushKV("bits", strprintf("%08x", pblock->nBits)); result.pushKV("height", (int64_t)(pindexPrev->nHeight+1)); + if (consensusParams.signet_blocks) { + result.pushKV("signet_challenge", HexStr(consensusParams.signet_challenge)); + } + if (!pblocktemplate->vchCoinbaseCommitment.empty()) { result.pushKV("default_witness_commitment", HexStr(pblocktemplate->vchCoinbaseCommitment)); } @@ -1027,21 +1042,19 @@ static RPCHelpMan submitheader() static RPCHelpMan estimatesmartfee() { return RPCHelpMan{"estimatesmartfee", - "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" - "confirmation within conf_target blocks if possible and return the number of blocks\n" - "for which the estimate is valid. Uses virtual transaction size as defined\n" - "in BIP 141 (witness data is discounted).\n", - { - {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "CONSERVATIVE", "The fee estimate mode.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within conf_target blocks if possible and return the number of blocks\n" + "for which the estimate is valid. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n", + { + {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "conservative", "The fee estimate mode.\n" " Whether to return a more conservative estimate which also satisfies\n" " a longer history. A conservative estimate potentially returns a\n" " higher feerate and is more likely to be sufficient for the desired\n" " target, but is not as responsive to short term drops in the\n" - " prevailing fee market. Must be one of:\n" - " \"UNSET\"\n" - " \"ECONOMICAL\"\n" - " \"CONSERVATIVE\""}, + " prevailing fee market. Must be one of (case insensitive):\n" + "\"" + FeeModes("\"\n\"") + "\""}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -1064,7 +1077,10 @@ static RPCHelpMan estimatesmartfee() { RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + + CBlockPolicyEstimator& fee_estimator = EnsureFeeEstimator(request.context); + + unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); bool conservative = true; if (!request.params[1].isNull()) { @@ -1078,7 +1094,7 @@ static RPCHelpMan estimatesmartfee() UniValue result(UniValue::VOBJ); UniValue errors(UniValue::VARR); FeeCalculation feeCalc; - CFeeRate feeRate = ::feeEstimator.estimateSmartFee(conf_target, &feeCalc, conservative); + CFeeRate feeRate = fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative); if (feeRate != CFeeRate(0)) { result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); } else { @@ -1149,7 +1165,10 @@ static RPCHelpMan estimaterawfee() { RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + + CBlockPolicyEstimator& fee_estimator = EnsureFeeEstimator(request.context); + + unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); double threshold = 0.95; if (!request.params[1].isNull()) { @@ -1161,14 +1180,14 @@ static RPCHelpMan estimaterawfee() UniValue result(UniValue::VOBJ); - for (const FeeEstimateHorizon horizon : {FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}) { + for (const FeeEstimateHorizon horizon : ALL_FEE_ESTIMATE_HORIZONS) { CFeeRate feeRate; EstimationResult buckets; // Only output results for horizons which track the target - if (conf_target > ::feeEstimator.HighestTargetTracked(horizon)) continue; + if (conf_target > fee_estimator.HighestTargetTracked(horizon)) continue; - feeRate = ::feeEstimator.estimateRawFee(conf_target, threshold, horizon, &buckets); + feeRate = fee_estimator.estimateRawFee(conf_target, threshold, horizon, &buckets); UniValue horizon_result(UniValue::VOBJ); UniValue errors(UniValue::VARR); UniValue passbucket(UniValue::VOBJ); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 0c982317f5..b3102a236d 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -624,8 +624,6 @@ static RPCHelpMan echo(const std::string& name) RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (request.fHelp) throw std::runtime_error(self.ToString()); - if (request.params[9].isStr()) { CHECK_NONFATAL(request.params[9].get_str() != "trigger_internal_bug"); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f98ea63782..cfca8b4ad4 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -5,6 +5,7 @@ #include <rpc/server.h> #include <banman.h> +#include <chainparams.h> #include <clientversion.h> #include <core_io.h> #include <net.h> @@ -126,21 +127,19 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, - {RPCResult::Type::BOOL, "addnode", "Whether connection was due to addnode/-connect or if it was an automatic/inbound connection\n" - "(DEPRECATED, returned only if the config option -deprecatedrpc=getpeerinfo_addnode is passed)"}, - {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" - "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" - "best capture connection behaviors."}, + {RPCResult::Type::BOOL, "bip152_hb_to", "Whether we selected peer as (compact blocks) high-bandwidth peer"}, + {RPCResult::Type::BOOL, "bip152_hb_from", "Whether peer selected us as (compact blocks) high-bandwidth peer"}, {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, - {RPCResult::Type::NUM, "banscore", "The ban score (DEPRECATED, returned only if config option -deprecatedrpc=banscore is passed)"}, {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, {RPCResult::Type::ARR, "inflight", "", { {RPCResult::Type::NUM, "n", "The heights of blocks we're currently asking from this peer"}, }}, - {RPCResult::Type::BOOL, "whitelisted", /* optional */ true, "Whether the peer is whitelisted with default permissions\n" - "(DEPRECATED, returned only if config option -deprecatedrpc=whitelisted is passed)"}, + {RPCResult::Type::ARR, "permissions", "Any special permissions that have been granted to this peer", + { + {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, + }}, {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { @@ -155,6 +154,9 @@ static RPCHelpMan getpeerinfo() "Only known message types can appear as keys in the object and all bytes received\n" "of unknown message types are listed under '"+NET_MESSAGE_COMMAND_OTHER+"'."} }}, + {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" + "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" + "best capture connection behaviors."}, }}, }}, }, @@ -165,8 +167,9 @@ static RPCHelpMan getpeerinfo() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { NodeContext& node = EnsureNodeContext(request.context); - if(!node.connman) + if(!node.connman || !node.peerman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } std::vector<CNodeStats> vstats; node.connman->GetNodeStats(vstats); @@ -176,7 +179,7 @@ static RPCHelpMan getpeerinfo() for (const CNodeStats& stats : vstats) { UniValue obj(UniValue::VOBJ); CNodeStateStats statestats; - bool fStateStats = GetNodeStateStats(stats.nodeid, statestats); + bool fStateStats = node.peerman->GetNodeStateStats(stats.nodeid, statestats); obj.pushKV("id", stats.nodeid); obj.pushKV("addr", stats.addrName); if (stats.addrBind.IsValid()) { @@ -185,7 +188,7 @@ static RPCHelpMan getpeerinfo() if (!(stats.addrLocal.empty())) { obj.pushKV("addrlocal", stats.addrLocal); } - obj.pushKV("network", stats.m_network); + obj.pushKV("network", GetNetworkName(stats.m_network)); if (stats.m_mapped_as != 0) { obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); } @@ -215,16 +218,10 @@ static RPCHelpMan getpeerinfo() // their ver message. obj.pushKV("subver", stats.cleanSubVer); obj.pushKV("inbound", stats.fInbound); - if (IsDeprecatedRPCEnabled("getpeerinfo_addnode")) { - // addnode is deprecated in v0.21 for removal in v0.22 - obj.pushKV("addnode", stats.m_manual_connection); - } - obj.pushKV("startingheight", stats.nStartingHeight); + obj.pushKV("bip152_hb_to", stats.m_bip152_highbandwidth_to); + obj.pushKV("bip152_hb_from", stats.m_bip152_highbandwidth_from); if (fStateStats) { - if (IsDeprecatedRPCEnabled("banscore")) { - // banscore is deprecated in v0.21 for removal in v0.22 - obj.pushKV("banscore", statestats.m_misbehavior_score); - } + obj.pushKV("startingheight", statestats.m_starting_height); obj.pushKV("synced_headers", statestats.nSyncHeight); obj.pushKV("synced_blocks", statestats.nCommonHeight); UniValue heights(UniValue::VARR); @@ -233,10 +230,6 @@ static RPCHelpMan getpeerinfo() } obj.pushKV("inflight", heights); } - if (IsDeprecatedRPCEnabled("whitelisted")) { - // whitelisted is deprecated in v0.21 for removal in v0.22 - obj.pushKV("whitelisted", stats.m_legacyWhitelisted); - } UniValue permissions(UniValue::VARR); for (const auto& permission : NetPermissions::ToStrings(stats.m_permissionFlags)) { permissions.push_back(permission); @@ -257,7 +250,7 @@ static RPCHelpMan getpeerinfo() recvPerMsgCmd.pushKV(i.first, i.second); } obj.pushKV("bytesrecv_per_msg", recvPerMsgCmd); - obj.pushKV("connection_type", stats.m_conn_type_string); + obj.pushKV("connection_type", ConnectionTypeAsString(stats.m_conn_type)); ret.push_back(obj); } @@ -288,10 +281,10 @@ static RPCHelpMan addnode() std::string strCommand; if (!request.params[1].isNull()) strCommand = request.params[1].get_str(); - if (request.fHelp || request.params.size() != 2 || - (strCommand != "onetry" && strCommand != "add" && strCommand != "remove")) + if (strCommand != "onetry" && strCommand != "add" && strCommand != "remove") { throw std::runtime_error( self.ToString()); + } NodeContext& node = EnsureNodeContext(request.context); if(!node.connman) @@ -322,6 +315,61 @@ static RPCHelpMan addnode() }; } +static RPCHelpMan addconnection() +{ + return RPCHelpMan{"addconnection", + "\nOpen an outbound connection to a specified node. This RPC is for testing only.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address and port to attempt connecting to."}, + {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open, either \"outbound-full-relay\" or \"block-relay-only\"."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + { RPCResult::Type::STR, "address", "Address of newly added connection." }, + { RPCResult::Type::STR, "connection_type", "Type of connection opened." }, + }}, + RPCExamples{ + HelpExampleCli("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"") + + HelpExampleRpc("addconnection", "\"192.168.0.6:8333\" \"outbound-full-relay\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (Params().NetworkIDString() != CBaseChainParams::REGTEST) { + throw std::runtime_error("addconnection is for regression testing (-regtest mode) only."); + } + + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VSTR}); + const std::string address = request.params[0].get_str(); + const std::string conn_type_in{TrimString(request.params[1].get_str())}; + ConnectionType conn_type{}; + if (conn_type_in == "outbound-full-relay") { + conn_type = ConnectionType::OUTBOUND_FULL_RELAY; + } else if (conn_type_in == "block-relay-only") { + conn_type = ConnectionType::BLOCK_RELAY; + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString()); + } + + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { + throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled."); + } + + const bool success = node.connman->AddConnection(address, conn_type); + if (!success) { + throw JSONRPCError(RPC_CLIENT_NODE_CAPACITY_REACHED, "Error: Already at capacity for specified connection type."); + } + + UniValue info(UniValue::VOBJ); + info.pushKV("address", address); + info.pushKV("connection_type", conn_type_in); + + return info; +}, + }; +} + static RPCHelpMan disconnectnode() { return RPCHelpMan{"disconnectnode", @@ -482,12 +530,12 @@ static RPCHelpMan getnettotals() obj.pushKV("timemillis", GetTimeMillis()); UniValue outboundLimit(UniValue::VOBJ); - outboundLimit.pushKV("timeframe", node.connman->GetMaxOutboundTimeframe()); + outboundLimit.pushKV("timeframe", count_seconds(node.connman->GetMaxOutboundTimeframe())); outboundLimit.pushKV("target", node.connman->GetMaxOutboundTarget()); outboundLimit.pushKV("target_reached", node.connman->OutboundTargetReached(false)); outboundLimit.pushKV("serve_historical_blocks", !node.connman->OutboundTargetReached(true)); outboundLimit.pushKV("bytes_left_in_cycle", node.connman->GetOutboundTargetBytesLeft()); - outboundLimit.pushKV("time_left_in_cycle", node.connman->GetMaxOutboundTimeLeftInCycle()); + outboundLimit.pushKV("time_left_in_cycle", count_seconds(node.connman->GetMaxOutboundTimeLeftInCycle())); obj.pushKV("uploadtarget", outboundLimit); return obj; }, @@ -577,7 +625,9 @@ static RPCHelpMan getnetworkinfo() obj.pushKV("localservices", strprintf("%016x", services)); obj.pushKV("localservicesnames", GetServicesNames(services)); } - obj.pushKV("localrelay", g_relay_txes); + if (node.peerman) { + obj.pushKV("localrelay", !node.peerman->IgnoresIncomingTxs()); + } obj.pushKV("timeoffset", GetTimeOffset()); if (node.connman) { obj.pushKV("networkactive", node.connman->GetNetworkActive()); @@ -628,7 +678,7 @@ static RPCHelpMan setban() std::string strCommand; if (!request.params[1].isNull()) strCommand = request.params[1].get_str(); - if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { + if (strCommand != "add" && strCommand != "remove") { throw std::runtime_error(help.ToString()); } NodeContext& node = EnsureNodeContext(request.context); @@ -906,6 +956,8 @@ static const CRPCCommand commands[] = { "network", "clearbanned", &clearbanned, {} }, { "network", "setnetworkactive", &setnetworkactive, {"state"} }, { "network", "getnodeaddresses", &getnodeaddresses, {"count"} }, + + { "hidden", "addconnection", &addconnection, {"address", "connection_type"} }, { "hidden", "addpeeraddress", &addpeeraddress, {"address", "port"} }, }; // clang-format on diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index d1475f452d..c8ceb2c186 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -62,6 +62,7 @@ enum RPCErrorCode RPC_CLIENT_NODE_NOT_CONNECTED = -29, //!< Node to disconnect not found in connected nodes RPC_CLIENT_INVALID_IP_OR_SUBNET = -30, //!< Invalid IP/Subnet RPC_CLIENT_P2P_DISABLED = -31, //!< No valid connection manager instance found + RPC_CLIENT_NODE_CAPACITY_REACHED= -34, //!< Max number of outbound or block-relay connections already open //! Chain errors RPC_CLIENT_MEMPOOL_DISABLED = -33, //!< No mempool instance found diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index c6d7fea443..e2897549b5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -109,7 +109,7 @@ static RPCHelpMan getrawtransaction() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::STR, "vout", ""}, + {RPCResult::Type::NUM, "vout", "The output number"}, {RPCResult::Type::OBJ, "scriptSig", "The script", { {RPCResult::Type::STR, "asm", "asm"}, @@ -244,16 +244,15 @@ static RPCHelpMan gettxoutproof() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::set<uint256> setTxids; - uint256 oneTxid; UniValue txids = request.params[0].get_array(); + if (txids.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); + } for (unsigned int idx = 0; idx < txids.size(); idx++) { - const UniValue& txid = txids[idx]; - uint256 hash(ParseHashV(txid, "txid")); - if (setTxids.count(hash)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txid.get_str()); + auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); + if (!ret.second) { + throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); } - setTxids.insert(hash); - oneTxid = hash; } CBlockIndex* pblockindex = nullptr; @@ -287,7 +286,7 @@ static RPCHelpMan gettxoutproof() LOCK(cs_main); if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, oneTxid, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, /* mempool */ nullptr, *setTxids.begin(), Params().GetConsensus(), hashBlock); if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index f004ecc20c..122a92f084 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -286,7 +286,7 @@ void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, SignTransactionResultToJSON(mtx, complete, coins, input_errors, result); } -void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, std::map<int, std::string>& input_errors, UniValue& result) +void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, std::string>& input_errors, UniValue& result) { // Make errors UniValue UniValue vErrors(UniValue::VARR); diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 942314eccf..ce7d5834fa 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -25,7 +25,7 @@ class SigningProvider; * @param result JSON object where signed transaction results accumulate */ void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType, UniValue& result); -void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, std::map<int, std::string>& input_errors, UniValue& result); +void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, std::string>& input_errors, UniValue& result); /** * Parse a prevtxs UniValue array and get the map of coins from it diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index d9ad70fa37..a7866474e1 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/request.h b/src/rpc/request.h index 4761e9e371..de3a4ae840 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -40,7 +40,7 @@ public: std::string peerAddr; const util::Ref& context; - JSONRPCRequest(const util::Ref& context) : id(NullUniValue), params(NullUniValue), fHelp(false), context(context) {} + explicit JSONRPCRequest(const util::Ref& context) : id(NullUniValue), params(NullUniValue), fHelp(false), context(context) {} //! Initializes request information from another request object and the //! given context. The implementation should be updated if any members are diff --git a/src/rpc/server.h b/src/rpc/server.h index b2358ac5b2..040192b455 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -85,7 +85,6 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); */ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nSeconds); -typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); typedef RPCHelpMan (*RpcMethodFnType)(); class CRPCCommand @@ -116,14 +115,6 @@ public: CHECK_NONFATAL(fn().GetArgNames() == args_in); } - //! Simplified constructor taking plain rpcfn_type function pointer. - CRPCCommand(const char* category, const char* name, rpcfn_type fn, std::initializer_list<const char*> args) - : CRPCCommand(category, name, - [fn](const JSONRPCRequest& request, UniValue& result, bool) { result = fn(request); return true; }, - {args.begin(), args.end()}, intptr_t(fn)) - { - } - std::string category; std::string name; Actor actor; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 1b21587b6d..31072114da 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Bitcoin Core developers +// Copyright (c) 2017-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -209,7 +209,7 @@ CTxDestination AddAndGetMultisigDestination(const int required, const std::vecto return dest; } -class DescribeAddressVisitor : public boost::static_visitor<UniValue> +class DescribeAddressVisitor { public: explicit DescribeAddressVisitor() {} @@ -267,7 +267,7 @@ public: UniValue DescribeAddress(const CTxDestination& dest) { - return boost::apply_visitor(DescribeAddressVisitor(), dest); + return std::visit(DescribeAddressVisitor(), dest); } unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) @@ -562,10 +562,10 @@ std::string RPCArg::GetName() const bool RPCArg::IsOptional() const { - if (m_fallback.which() == 1) { + if (m_fallback.index() == 1) { return true; } else { - return RPCArg::Optional::NO != boost::get<RPCArg::Optional>(m_fallback); + return RPCArg::Optional::NO != std::get<RPCArg::Optional>(m_fallback); } } @@ -609,10 +609,10 @@ std::string RPCArg::ToDescriptionString() const } } // no default case, so the compiler can warn about missing cases } - if (m_fallback.which() == 1) { - ret += ", optional, default=" + boost::get<std::string>(m_fallback); + if (m_fallback.index() == 1) { + ret += ", optional, default=" + std::get<std::string>(m_fallback); } else { - switch (boost::get<RPCArg::Optional>(m_fallback)) { + switch (std::get<RPCArg::Optional>(m_fallback)) { case RPCArg::Optional::OMITTED: { // nothing to do. Element is treated as if not present and has no default value break; diff --git a/src/rpc/util.h b/src/rpc/util.h index 45b0bb0c7e..942c243718 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Bitcoin Core developers +// Copyright (c) 2017-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,10 +19,9 @@ #include <util/check.h> #include <string> +#include <variant> #include <vector> -#include <boost/variant.hpp> - /** * String used to describe UNIX epoch time in documentation, factored out to a * constant for consistency. @@ -144,7 +143,7 @@ struct RPCArg { */ OMITTED, }; - using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; + using Fallback = std::variant<Optional, /* default value for optional args */ std::string>; const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; const bool m_hidden; diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 15e204062f..76609f01a7 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -16,8 +16,7 @@ namespace { class TxInputStream { public: - TxInputStream(int nTypeIn, int nVersionIn, const unsigned char *txTo, size_t txToLen) : - m_type(nTypeIn), + TxInputStream(int nVersionIn, const unsigned char *txTo, size_t txToLen) : m_version(nVersionIn), m_data(txTo), m_remaining(txToLen) @@ -47,9 +46,7 @@ public: } int GetVersion() const { return m_version; } - int GetType() const { return m_type; } private: - const int m_type; const int m_version; const unsigned char* m_data; size_t m_remaining; @@ -84,7 +81,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP return set_error(err, bitcoinconsensus_ERR_INVALID_FLAGS); } try { - TxInputStream stream(SER_NETWORK, PROTOCOL_VERSION, txTo, txToLen); + TxInputStream stream(PROTOCOL_VERSION, txTo, txToLen); CTransaction tx(deserialize, stream); if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 6c0a98cca2..9e4b8a9dd6 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -156,7 +156,7 @@ protected: uint32_t m_expr_index; public: - PubkeyProvider(uint32_t exp_index) : m_expr_index(exp_index) {} + explicit PubkeyProvider(uint32_t exp_index) : m_expr_index(exp_index) {} virtual ~PubkeyProvider() = default; @@ -565,7 +565,7 @@ public: Optional<OutputType> GetOutputType() const override { - switch (m_destination.which()) { + switch (m_destination.index()) { case 1 /* PKHash */: case 2 /* ScriptHash */: return OutputType::LEGACY; case 3 /* WitnessV0ScriptHash */: @@ -593,7 +593,7 @@ public: { CTxDestination dest; ExtractDestination(m_script, dest); - switch (dest.which()) { + switch (dest.index()) { case 1 /* PKHash */: case 2 /* ScriptHash */: return OutputType::LEGACY; case 3 /* WitnessV0ScriptHash */: @@ -731,7 +731,7 @@ enum class ParseScriptContext { }; /** Parse a key path, being passed a split list of elements (the first element is ignored). */ -NODISCARD bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, std::string& error) +[[nodiscard]] bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, std::string& error) { for (size_t i = 1; i < split.size(); ++i) { Span<const char> elem = split[i]; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 5735e7df66..ecac3b9e7e 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -305,8 +305,8 @@ private: uint32_t m_first_false_pos = NO_FALSE; public: - bool empty() { return m_stack_size == 0; } - bool all_true() { return m_first_false_pos == NO_FALSE; } + bool empty() const { return m_stack_size == 0; } + bool all_true() const { return m_first_false_pos == NO_FALSE; } void push_back(bool f) { if (m_first_false_pos == NO_FALSE && !f) { @@ -1834,9 +1834,13 @@ static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CS static bool VerifyTaprootCommitment(const std::vector<unsigned char>& control, const std::vector<unsigned char>& program, const CScript& script, uint256& tapleaf_hash) { const int path_len = (control.size() - TAPROOT_CONTROL_BASE_SIZE) / TAPROOT_CONTROL_NODE_SIZE; + //! The inner pubkey (x-only, so no Y coordinate parity). const XOnlyPubKey p{uint256(std::vector<unsigned char>(control.begin() + 1, control.begin() + TAPROOT_CONTROL_BASE_SIZE))}; + //! The output pubkey (taken from the scriptPubKey). const XOnlyPubKey q{uint256(program)}; + // Compute the tapleaf hash. tapleaf_hash = (CHashWriter(HASHER_TAPLEAF) << uint8_t(control[0] & TAPROOT_LEAF_MASK) << script).GetSHA256(); + // Compute the Merkle root from the leaf and the provided path. uint256 k = tapleaf_hash; for (int i = 0; i < path_len; ++i) { CHashWriter ss_branch{HASHER_TAPBRANCH}; @@ -1848,7 +1852,9 @@ static bool VerifyTaprootCommitment(const std::vector<unsigned char>& control, c } k = ss_branch.GetSHA256(); } + // Compute the tweak from the Merkle root and the inner pubkey. k = (CHashWriter(HASHER_TAPTWEAK) << MakeSpan(p) << k).GetSHA256(); + // Verify that the output pubkey matches the tweaked inner pubkey, after correcting for parity. return q.CheckPayToContract(p, k, control[0] & 1); } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index c0c2b012c6..b4c163c841 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/keyorigin.h b/src/script/keyorigin.h index a318ff0f9d..210395d177 100644 --- a/src/script/keyorigin.h +++ b/src/script/keyorigin.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script.cpp b/src/script/script.cpp index f31472e42d..9a6419088b 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script_error.h b/src/script/script_error.h index b071681613..44e68fe0fa 100644 --- a/src/script/script_error.h +++ b/src/script/script_error.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index c1786140de..2bfe206597 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -66,7 +66,7 @@ public: return setValid.contains(entry, erase); } - void Set(uint256& entry) + void Set(const uint256& entry) { boost::unique_lock<boost::shared_mutex> lock(cs_sigcache); setValid.insert(entry); diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 00534f9758..bf0ba38c2d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <script/interpreter.h> #include <span.h> +#include <util/hasher.h> #include <vector> @@ -20,27 +21,6 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 0e6864d547..8afbe9ebed 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -388,7 +388,7 @@ bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, C bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, unsigned int nIn, int nHashType) { assert(nIn < txTo.vin.size()); - CTxIn& txin = txTo.vin[nIn]; + const CTxIn& txin = txTo.vin[nIn]; assert(txin.prevout.n < txFrom.vout.size()); const CTxOut& txout = txFrom.vout[txin.prevout.n]; diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index 2d8dc7d471..9781ec32af 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -179,18 +179,18 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& { // Only supports destinations which map to single public keys, i.e. P2PKH, // P2WPKH, and P2SH-P2WPKH. - if (auto id = boost::get<PKHash>(&dest)) { + if (auto id = std::get_if<PKHash>(&dest)) { return ToKeyID(*id); } - if (auto witness_id = boost::get<WitnessV0KeyHash>(&dest)) { + if (auto witness_id = std::get_if<WitnessV0KeyHash>(&dest)) { return ToKeyID(*witness_id); } - if (auto script_hash = boost::get<ScriptHash>(&dest)) { + if (auto script_hash = std::get_if<ScriptHash>(&dest)) { CScript script; CScriptID script_id(*script_hash); CTxDestination inner_dest; if (store.GetCScript(script_id, script) && ExtractDestination(script, inner_dest)) { - if (auto inner_witness_id = boost::get<WitnessV0KeyHash>(&inner_dest)) { + if (auto inner_witness_id = std::get_if<WitnessV0KeyHash>(&inner_dest)) { return ToKeyID(*inner_witness_id); } } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index f2f81664f6..7967c01858 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -261,9 +261,8 @@ bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::v return true; } -namespace -{ -class CScriptVisitor : public boost::static_visitor<CScript> +namespace { +class CScriptVisitor { public: CScript operator()(const CNoDestination& dest) const @@ -300,7 +299,7 @@ public: CScript GetScriptForDestination(const CTxDestination& dest) { - return boost::apply_visitor(CScriptVisitor(), dest); + return std::visit(CScriptVisitor(), dest); } CScript GetScriptForRawPubKey(const CPubKey& pubKey) @@ -320,5 +319,5 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) } bool IsValidDestination(const CTxDestination& dest) { - return dest.which() != 0; + return dest.index() != 0; } diff --git a/src/script/standard.h b/src/script/standard.h index 721203385e..d5d87392ad 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -9,10 +9,8 @@ #include <script/interpreter.h> #include <uint256.h> -#include <boost/variant.hpp> - #include <string> - +#include <variant> static const bool DEFAULT_ACCEPT_DATACARRIER = true; @@ -28,7 +26,7 @@ protected: public: BaseHash() : m_hash() {} - BaseHash(const HashType& in) : m_hash(in) {} + explicit BaseHash(const HashType& in) : m_hash(in) {} unsigned char* begin() { @@ -211,7 +209,7 @@ struct WitnessUnknown * (taproot outputs do not require their own type as long as no wallet support exists) * A CTxDestination is the internal data type encoded in a bitcoin address */ -typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown> CTxDestination; +using CTxDestination = std::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessUnknown>; /** Check whether a CTxDestination is a CNoDestination. */ bool IsValidDestination(const CTxDestination& dest); diff --git a/src/shutdown.cpp b/src/shutdown.cpp index dec497d8ec..df5f996022 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -1,23 +1,112 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <shutdown.h> +#include <config/bitcoin-config.h> + +#include <assert.h> #include <atomic> +#ifdef WIN32 +#include <condition_variable> +#else +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#endif static std::atomic<bool> fRequestShutdown(false); +#ifdef WIN32 +/** On windows it is possible to simply use a condition variable. */ +std::mutex g_shutdown_mutex; +std::condition_variable g_shutdown_cv; +#else +/** On UNIX-like operating systems use the self-pipe trick. + * Index 0 will be the read end of the pipe, index 1 the write end. + */ +static int g_shutdown_pipe[2] = {-1, -1}; +#endif + +bool InitShutdownState() +{ +#ifndef WIN32 +#if HAVE_O_CLOEXEC + // If we can, make sure that the file descriptors are closed on exec() + // to prevent interference. + if (pipe2(g_shutdown_pipe, O_CLOEXEC) != 0) { + return false; + } +#else + if (pipe(g_shutdown_pipe) != 0) { + return false; + } +#endif +#endif + return true; +} void StartShutdown() { +#ifdef WIN32 + std::unique_lock<std::mutex> lk(g_shutdown_mutex); fRequestShutdown = true; + g_shutdown_cv.notify_one(); +#else + // This must be reentrant and safe for calling in a signal handler, so using a condition variable is not safe. + // Make sure that the token is only written once even if multiple threads call this concurrently or in + // case of a reentrant signal. + if (!fRequestShutdown.exchange(true)) { + // Write an arbitrary byte to the write end of the shutdown pipe. + const char token = 'x'; + while (true) { + int result = write(g_shutdown_pipe[1], &token, 1); + if (result < 0) { + // Failure. It's possible that the write was interrupted by another signal. + // Other errors are unexpected here. + assert(errno == EINTR); + } else { + assert(result == 1); + break; + } + } + } +#endif } + void AbortShutdown() { + if (fRequestShutdown) { + // Cancel existing shutdown by waiting for it, this will reset condition flags and remove + // the shutdown token from the pipe. + WaitForShutdown(); + } fRequestShutdown = false; } + bool ShutdownRequested() { return fRequestShutdown; } + +void WaitForShutdown() +{ +#ifdef WIN32 + std::unique_lock<std::mutex> lk(g_shutdown_mutex); + g_shutdown_cv.wait(lk, [] { return fRequestShutdown.load(); }); +#else + char token; + while (true) { + int result = read(g_shutdown_pipe[0], &token, 1); + if (result < 0) { + // Failure. Check if the read was interrupted by a signal. + // Other errors are unexpected here. + assert(errno == EINTR); + } else { + assert(result == 1); + break; + } + } +#endif +} diff --git a/src/shutdown.h b/src/shutdown.h index 3ed851c789..b2fbdb8cfb 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -1,13 +1,30 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SHUTDOWN_H #define BITCOIN_SHUTDOWN_H +/** Initialize shutdown state. This must be called before using either StartShutdown(), + * AbortShutdown() or WaitForShutdown(). Calling ShutdownRequested() is always safe. + */ +bool InitShutdownState(); + +/** Request shutdown of the application. */ void StartShutdown(); + +/** Clear shutdown flag. Only use this during init (before calling WaitForShutdown in any + * thread), or in the unit tests. Calling it in other circumstances will cause a race condition. + */ void AbortShutdown(); + +/** Returns true if a shutdown is requested, false otherwise. */ bool ShutdownRequested(); +/** Wait for StartShutdown to be called in any thread. This can only be used + * from a single thread. + */ +void WaitForShutdown(); + #endif diff --git a/src/span.h b/src/span.h index 4afb383a59..830164514b 100644 --- a/src/span.h +++ b/src/span.h @@ -18,6 +18,16 @@ #define ASSERT_IF_DEBUG(x) #endif +#if defined(__clang__) +#if __has_attribute(lifetimebound) +#define SPAN_ATTR_LIFETIMEBOUND [[clang::lifetimebound]] +#else +#define SPAN_ATTR_LIFETIMEBOUND +#endif +#else +#define SPAN_ATTR_LIFETIMEBOUND +#endif + /** A Span is an object that can refer to a contiguous sequence of objects. * * It implements a subset of C++20's std::span. @@ -84,6 +94,14 @@ class Span C* m_data; std::size_t m_size; + template <class T> + struct is_Span_int : public std::false_type {}; + template <class T> + struct is_Span_int<Span<T>> : public std::true_type {}; + template <class T> + struct is_Span : public is_Span_int<typename std::remove_cv<T>::type>{}; + + public: constexpr Span() noexcept : m_data(nullptr), m_size(0) {} @@ -134,8 +152,19 @@ public: * To prevent surprises, only Spans for constant value types are supported when passing in temporaries. * Note that this restriction does not exist when converting arrays or other Spans (see above). */ - template <typename V, typename std::enable_if<(std::is_const<C>::value || std::is_lvalue_reference<V>::value) && std::is_convertible<typename std::remove_pointer<decltype(std::declval<V&>().data())>::type (*)[], C (*)[]>::value && std::is_convertible<decltype(std::declval<V&>().size()), std::size_t>::value, int>::type = 0> - constexpr Span(V&& v) noexcept : m_data(v.data()), m_size(v.size()) {} + template <typename V> + constexpr Span(V& other SPAN_ATTR_LIFETIMEBOUND, + typename std::enable_if<!is_Span<V>::value && + std::is_convertible<typename std::remove_pointer<decltype(std::declval<V&>().data())>::type (*)[], C (*)[]>::value && + std::is_convertible<decltype(std::declval<V&>().size()), std::size_t>::value, std::nullptr_t>::type = nullptr) + : m_data(other.data()), m_size(other.size()){} + + template <typename V> + constexpr Span(const V& other SPAN_ATTR_LIFETIMEBOUND, + typename std::enable_if<!is_Span<V>::value && + std::is_convertible<typename std::remove_pointer<decltype(std::declval<const V&>().data())>::type (*)[], C (*)[]>::value && + std::is_convertible<decltype(std::declval<const V&>().size()), std::size_t>::value, std::nullptr_t>::type = nullptr) + : m_data(other.data()), m_size(other.size()){} constexpr C* data() const noexcept { return m_data; } constexpr C* begin() const noexcept { return m_data; } @@ -192,9 +221,9 @@ public: /** MakeSpan for arrays: */ template <typename A, int N> Span<A> constexpr MakeSpan(A (&a)[N]) { return Span<A>(a, N); } /** MakeSpan for temporaries / rvalue references, only supporting const output. */ -template <typename V> constexpr auto MakeSpan(V&& v) -> typename std::enable_if<!std::is_lvalue_reference<V>::value, Span<const typename std::remove_pointer<decltype(v.data())>::type>>::type { return std::forward<V>(v); } +template <typename V> constexpr auto MakeSpan(V&& v SPAN_ATTR_LIFETIMEBOUND) -> typename std::enable_if<!std::is_lvalue_reference<V>::value, Span<const typename std::remove_pointer<decltype(v.data())>::type>>::type { return std::forward<V>(v); } /** MakeSpan for (lvalue) references, supporting mutable output. */ -template <typename V> constexpr auto MakeSpan(V& v) -> Span<typename std::remove_pointer<decltype(v.data())>::type> { return v; } +template <typename V> constexpr auto MakeSpan(V& v SPAN_ATTR_LIFETIMEBOUND) -> Span<typename std::remove_pointer<decltype(v.data())>::type> { return v; } /** Pop the last element off a span, and return a reference to that element. */ template <typename T> diff --git a/src/streams.h b/src/streams.h index c22f5936fd..fce411d3df 100644 --- a/src/streams.h +++ b/src/streams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 26de780f29..6965f40253 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -65,7 +65,7 @@ void* Arena::alloc(size_t size) // Pick a large enough free-chunk. Returns an iterator pointing to the first element that is not less than key. // This allocation strategy is best-fit. According to "Dynamic Storage Allocation: A Survey and Critical Review", - // Wilson et. al. 1995, http://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf, best-fit and first-fit + // Wilson et. al. 1995, https://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf, best-fit and first-fit // policies seem to work well in practice. auto size_ptr_it = size_to_free_chunk.lower_bound(size); if (size_ptr_it == size_to_free_chunk.end()) diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index b9e2e99d1a..03e4e371a3 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.cpp b/src/sync.cpp index 322198a852..acfbe8fe29 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -13,10 +13,14 @@ #include <util/strencodings.h> #include <util/threadnames.h> +#include <boost/thread/mutex.hpp> + #include <map> +#include <mutex> #include <set> #include <system_error> #include <thread> +#include <type_traits> #include <unordered_map> #include <utility> #include <vector> @@ -135,16 +139,52 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac throw std::logic_error(strprintf("potential deadlock detected: %s -> %s -> %s", mutex_b, mutex_a, mutex_b)); } -static void push_lock(void* c, const CLockLocation& locklocation) +static void double_lock_detected(const void* mutex, const LockStack& lock_stack) +{ + LogPrintf("DOUBLE LOCK DETECTED\n"); + LogPrintf("Lock order:\n"); + for (const LockStackItem& i : lock_stack) { + if (i.first == mutex) { + LogPrintf(" (*)"); /* Continued */ + } + LogPrintf(" %s\n", i.second.ToString()); + } + if (g_debug_lockorder_abort) { + tfm::format(std::cerr, + "Assertion failed: detected double lock for %s, details in debug log.\n", + lock_stack.back().second.ToString()); + abort(); + } + throw std::logic_error("double lock detected"); +} + +template <typename MutexType> +static void push_lock(MutexType* c, const CLockLocation& locklocation) { + constexpr bool is_recursive_mutex = + std::is_base_of<RecursiveMutex, MutexType>::value || + std::is_base_of<std::recursive_mutex, MutexType>::value; + LockData& lockdata = GetLockData(); std::lock_guard<std::mutex> lock(lockdata.dd_mutex); LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; lock_stack.emplace_back(c, locklocation); - for (const LockStackItem& i : lock_stack) { - if (i.first == c) - break; + for (size_t j = 0; j < lock_stack.size() - 1; ++j) { + const LockStackItem& i = lock_stack[j]; + if (i.first == c) { + if (is_recursive_mutex) { + break; + } + // It is not a recursive mutex and it appears in the stack two times: + // at position `j` and at the end (which we added just before this loop). + // Can't allow locking the same (non-recursive) mutex two times from the + // same thread as that results in an undefined behavior. + auto lock_stack_copy = lock_stack; + lock_stack.pop_back(); + double_lock_detected(c, lock_stack_copy); + // double_lock_detected() does not return. + } const LockPair p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) @@ -175,27 +215,41 @@ static void pop_lock() } } -void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) +template <typename MutexType> +void EnterCritical(const char* pszName, const char* pszFile, int nLine, MutexType* cs, bool fTry) { push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry, util::ThreadGetInternalName())); } +template void EnterCritical(const char*, const char*, int, Mutex*, bool); +template void EnterCritical(const char*, const char*, int, RecursiveMutex*, bool); +template void EnterCritical(const char*, const char*, int, std::mutex*, bool); +template void EnterCritical(const char*, const char*, int, std::recursive_mutex*, bool); +template void EnterCritical(const char*, const char*, int, boost::mutex*, bool); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { - { - LockData& lockdata = GetLockData(); - std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - - const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; - if (!lock_stack.empty()) { - const auto& lastlock = lock_stack.back(); - if (lastlock.first == cs) { - lockname = lastlock.second.Name(); - return; - } + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + if (!lock_stack.empty()) { + const auto& lastlock = lock_stack.back(); + if (lastlock.first == cs) { + lockname = lastlock.second.Name(); + return; } } - throw std::system_error(EPERM, std::generic_category(), strprintf("%s:%s %s was not most recent critical section locked", file, line, guardname)); + + LogPrintf("INCONSISTENT LOCK ORDER DETECTED\n"); + LogPrintf("Current lock order (least recent first) is:\n"); + for (const LockStackItem& i : lock_stack) { + LogPrintf(" %s\n", i.second.ToString()); + } + if (g_debug_lockorder_abort) { + tfm::format(std::cerr, "%s:%s %s was not most recent critical section locked, details in debug log.\n", file, line, guardname); + abort(); + } + throw std::logic_error(strprintf("%s was not most recent critical section locked", guardname)); } void LeaveCritical() diff --git a/src/sync.h b/src/sync.h index 41f4e43bdd..53213c2089 100644 --- a/src/sync.h +++ b/src/sync.h @@ -48,7 +48,8 @@ LEAVE_CRITICAL_SECTION(mutex); // no RAII /////////////////////////////// #ifdef DEBUG_LOCKORDER -void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false); +template <typename MutexType> +void EnterCritical(const char* pszName, const char* pszFile, int nLine, MutexType* cs, bool fTry = false); void LeaveCritical(); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line); std::string LocksHeld(); @@ -66,7 +67,8 @@ bool LockStackEmpty(); */ extern bool g_debug_lockorder_abort; #else -inline void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} +template <typename MutexType> +inline void EnterCritical(const char* pszName, const char* pszFile, int nLine, MutexType* cs, bool fTry = false) {} inline void LeaveCritical() {} inline void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {} template <typename MutexType> @@ -135,7 +137,7 @@ class SCOPED_LOCKABLE UniqueLock : public Base private: void Enter(const char* pszName, const char* pszFile, int nLine) { - EnterCritical(pszName, pszFile, nLine, (void*)(Base::mutex())); + EnterCritical(pszName, pszFile, nLine, Base::mutex()); #ifdef DEBUG_LOCKCONTENTION if (!Base::try_lock()) { PrintLockContention(pszName, pszFile, nLine); @@ -148,7 +150,7 @@ private: bool TryEnter(const char* pszName, const char* pszFile, int nLine) { - EnterCritical(pszName, pszFile, nLine, (void*)(Base::mutex()), true); + EnterCritical(pszName, pszFile, nLine, Base::mutex(), true); Base::try_lock(); if (!Base::owns_lock()) LeaveCritical(); @@ -205,7 +207,7 @@ public: ~reverse_lock() { templock.swap(lock); - EnterCritical(lockname.c_str(), file.c_str(), line, (void*)lock.mutex()); + EnterCritical(lockname.c_str(), file.c_str(), line, lock.mutex()); lock.lock(); } @@ -236,14 +238,16 @@ using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove #define ENTER_CRITICAL_SECTION(cs) \ { \ - EnterCritical(#cs, __FILE__, __LINE__, (void*)(&cs)); \ + EnterCritical(#cs, __FILE__, __LINE__, &cs); \ (cs).lock(); \ } -#define LEAVE_CRITICAL_SECTION(cs) \ - { \ - (cs).unlock(); \ - LeaveCritical(); \ +#define LEAVE_CRITICAL_SECTION(cs) \ + { \ + std::string lockname; \ + CheckLastCritical((void*)(&cs), lockname, #cs, __FILE__, __LINE__); \ + (cs).unlock(); \ + LeaveCritical(); \ } //! Run code while locking a mutex. @@ -254,7 +258,22 @@ using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove //! //! int val = WITH_LOCK(cs, return shared_val); //! -#define WITH_LOCK(cs, code) [&] { LOCK(cs); code; }() +//! Note: +//! +//! Since the return type deduction follows that of decltype(auto), while the +//! deduced type of: +//! +//! WITH_LOCK(cs, return {int i = 1; return i;}); +//! +//! is int, the deduced type of: +//! +//! WITH_LOCK(cs, return {int j = 1; return (j);}); +//! +//! is &int, a reference to a local variable +//! +//! The above is detectable at compile-time with the -Wreturn-local-addr flag in +//! gcc and the -Wreturn-stack-address flag in clang, both enabled by default. +#define WITH_LOCK(cs, code) [&]() -> decltype(auto) { LOCK(cs); code; }() class CSemaphore { diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 25fdd64568..37ff8a9afe 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -71,7 +71,7 @@ public: } // Simulates connection failure so that we can test eviction of offline nodes - void SimConnFail(CService& addr) + void SimConnFail(const CService& addr) { LOCK(cs); int64_t nLastSuccess = 1; diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index c16519a6b1..1a39498899 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index d519eca859..3f7c5e99ee 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -6,6 +6,9 @@ #include <util/strencodings.h> #include <boost/test/unit_test.hpp> +#include <string> + +using namespace std::literals; BOOST_FIXTURE_TEST_SUITE(base32_tests, BasicTestingSetup) @@ -26,14 +29,14 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) // Decoding strings with embedded NUL characters should fail bool failure; - (void)DecodeBase32(std::string("invalid", 7), &failure); - BOOST_CHECK_EQUAL(failure, true); - (void)DecodeBase32(std::string("AWSX3VPP", 8), &failure); - BOOST_CHECK_EQUAL(failure, false); - (void)DecodeBase32(std::string("AWSX3VPP\0invalid", 16), &failure); - BOOST_CHECK_EQUAL(failure, true); - (void)DecodeBase32(std::string("AWSX3VPPinvalid", 15), &failure); - BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase32("invalid\0"s, &failure); // correct size, invalid due to \0 + BOOST_CHECK(failure); + (void)DecodeBase32("AWSX3VPP"s, &failure); // valid + BOOST_CHECK(!failure); + (void)DecodeBase32("AWSX3VPP\0invalid"s, &failure); // correct size, invalid due to \0 + BOOST_CHECK(failure); + (void)DecodeBase32("AWSX3VPPinvalid"s, &failure); // invalid size + BOOST_CHECK(failure); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 6a636f2574..dd760fe999 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,9 @@ #include <univalue.h> #include <boost/test/unit_test.hpp> +#include <string> +using namespace std::literals; extern UniValue read_json(const std::string& jsondata); @@ -58,14 +60,14 @@ BOOST_AUTO_TEST_CASE(base58_DecodeBase58) BOOST_CHECK_MESSAGE(result.size() == expected.size() && std::equal(result.begin(), result.end(), expected.begin()), strTest); } - BOOST_CHECK(!DecodeBase58("invalid", result, 100)); - BOOST_CHECK(!DecodeBase58(std::string("invalid"), result, 100)); - BOOST_CHECK(!DecodeBase58(std::string("\0invalid", 8), result, 100)); + BOOST_CHECK(!DecodeBase58("invalid"s, result, 100)); + BOOST_CHECK(!DecodeBase58("invalid\0"s, result, 100)); + BOOST_CHECK(!DecodeBase58("\0invalid"s, result, 100)); - BOOST_CHECK(DecodeBase58(std::string("good", 4), result, 100)); - BOOST_CHECK(!DecodeBase58(std::string("bad0IOl", 7), result, 100)); - BOOST_CHECK(!DecodeBase58(std::string("goodbad0IOl", 11), result, 100)); - BOOST_CHECK(!DecodeBase58(std::string("good\0bad0IOl", 12), result, 100)); + BOOST_CHECK(DecodeBase58("good"s, result, 100)); + BOOST_CHECK(!DecodeBase58("bad0IOl"s, result, 100)); + BOOST_CHECK(!DecodeBase58("goodbad0IOl"s, result, 100)); + BOOST_CHECK(!DecodeBase58("good\0bad0IOl"s, result, 100)); // check that DecodeBase58 skips whitespace, but still fails with unexpected non-whitespace at the end. BOOST_CHECK(!DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t a", result, 3)); @@ -73,10 +75,10 @@ BOOST_AUTO_TEST_CASE(base58_DecodeBase58) std::vector<unsigned char> expected = ParseHex("971a55"); BOOST_CHECK_EQUAL_COLLECTIONS(result.begin(), result.end(), expected.begin(), expected.end()); - BOOST_CHECK(DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh", 21), result, 100)); - BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oi", 21), result, 100)); - BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh0IOl", 25), result, 100)); - BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh\00IOl", 26), result, 100)); + BOOST_CHECK(DecodeBase58Check("3vQB7B6MrGQZaxCuFg4oh"s, result, 100)); + BOOST_CHECK(!DecodeBase58Check("3vQB7B6MrGQZaxCuFg4oi"s, result, 100)); + BOOST_CHECK(!DecodeBase58Check("3vQB7B6MrGQZaxCuFg4oh0IOl"s, result, 100)); + BOOST_CHECK(!DecodeBase58Check("3vQB7B6MrGQZaxCuFg4oh\0" "0IOl"s, result, 100)); } BOOST_AUTO_TEST_CASE(base58_random_encode_decode) diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 5927eab6cf..bb8d102bd0 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -6,6 +6,9 @@ #include <util/strencodings.h> #include <boost/test/unit_test.hpp> +#include <string> + +using namespace std::literals; BOOST_FIXTURE_TEST_SUITE(base64_tests, BasicTestingSetup) @@ -23,14 +26,14 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) // Decoding strings with embedded NUL characters should fail bool failure; - (void)DecodeBase64(std::string("invalid", 7), &failure); - BOOST_CHECK_EQUAL(failure, true); - (void)DecodeBase64(std::string("nQB/pZw=", 8), &failure); - BOOST_CHECK_EQUAL(failure, false); - (void)DecodeBase64(std::string("nQB/pZw=\0invalid", 16), &failure); - BOOST_CHECK_EQUAL(failure, true); - (void)DecodeBase64(std::string("nQB/pZw=invalid", 15), &failure); - BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase64("invalid\0"s, &failure); + BOOST_CHECK(failure); + (void)DecodeBase64("nQB/pZw="s, &failure); + BOOST_CHECK(!failure); + (void)DecodeBase64("nQB/pZw=\0invalid"s, &failure); + BOOST_CHECK(failure); + (void)DecodeBase64("nQB/pZw=invalid\0"s, &failure); + BOOST_CHECK(failure); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 14cf1a4a76..9dfbd7ba7c 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 178c261365..8eb4dbc592 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 8348810ac1..5dbf07b420 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -26,7 +26,7 @@ static const unsigned int QUEUE_BATCH_SIZE = 128; static const int SCRIPT_CHECK_THREADS = 3; struct FakeCheck { - bool operator()() + bool operator()() const { return true; } @@ -47,7 +47,7 @@ struct FailingCheck { bool fails; FailingCheck(bool _fails) : fails(_fails){}; FailingCheck() : fails(true){}; - bool operator()() + bool operator()() const { return !fails; } @@ -76,7 +76,7 @@ struct UniqueCheck { struct MemoryCheck { static std::atomic<size_t> fake_allocated_memory; bool b {false}; - bool operator()() + bool operator()() const { return true; } @@ -107,7 +107,7 @@ struct FrozenCleanupCheck { // Freezing can't be the default initialized behavior given how the queue // swaps in default initialized Checks. bool should_freeze {false}; - bool operator()() + bool operator()() const { return true; } diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 173ec5e3d9..06db3b846e 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,7 +38,7 @@ class CCoinsViewTest : public CCoinsView std::map<COutPoint, Coin> map_; public: - NODISCARD bool GetCoin(const COutPoint& outpoint, Coin& coin) const override + [[nodiscard]] bool GetCoin(const COutPoint& outpoint, Coin& coin) const override { std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); if (it == map_.end()) { diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 0ad5066603..7358b246b6 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -14,7 +14,9 @@ #include <crypto/sha256.h> #include <crypto/sha3.h> #include <crypto/sha512.h> +#include <crypto/muhash.h> #include <random.h> +#include <streams.h> #include <test/util/setup_common.h> #include <util/strencodings.h> @@ -857,4 +859,92 @@ BOOST_AUTO_TEST_CASE(sha3_256_tests) TestSHA3_256("72c57c359e10684d0517e46653a02d18d29eff803eb009e4d5eb9e95add9ad1a4ac1f38a70296f3a369a16985ca3c957de2084cdc9bdd8994eb59b8815e0debad4ec1f001feac089820db8becdaf896aaf95721e8674e5d476b43bd2b873a7d135cd685f545b438210f9319e4dcd55986c85303c1ddf18dc746fe63a409df0a998ed376eb683e16c09e6e9018504152b3e7628ef350659fb716e058a5263a18823d2f2f6ee6a8091945a48ae1c5cb1694cf2c1fe76ef9177953afe8899cfa2b7fe0603bfa3180937dadfb66fbbdd119bbf8063338aa4a699075a3bfdbae8db7e5211d0917e9665a702fc9b0a0a901d08bea97654162d82a9f05622b060b634244779c33427eb7a29353a5f48b07cbefa72f3622ac5900bef77b71d6b314296f304c8426f451f32049b1f6af156a9dab702e8907d3cd72bb2c50493f4d593e731b285b70c803b74825b3524cda3205a8897106615260ac93c01c5ec14f5b11127783989d1824527e99e04f6a340e827b559f24db9292fcdd354838f9339a5fa1d7f6b2087f04835828b13463dd40927866f16ae33ed501ec0e6c4e63948768c5aeea3e4f6754985954bea7d61088c44430204ef491b74a64bde1358cecb2cad28ee6a3de5b752ff6a051104d88478653339457ac45ba44cbb65f54d1969d047cda746931d5e6a8b48e211416aefd5729f3d60b56b54e7f85aa2f42de3cb69419240c24e67139a11790a709edef2ac52cf35dd0a08af45926ebe9761f498ff83bfe263d6897ee97943a4b982fe3404ef0b4a45e06113c60340e0664f14799bf59cb4b3934b465fabefd87155905ee5309ba41e9e402973311831ea600b16437f71df39ee77130490c4d0227e5d1757fdc66af3ae6b9953053ed9aafca0160209858a7d4dd38fe10e0cb153672d08633ed6c54977aa0a6e67f9ff2f8c9d22dd7b21de08192960fd0e0da68d77c8d810db11dcaa61c725cd4092cbff76c8e1debd8d0361bb3f2e607911d45716f53067bdc0d89dd4889177765166a424e9fc0cb711201099dda213355e6639ac7eb86eca2ae0ab38b7f674f37ef8a6fcca1a6f52f55d9e1dcd631d2c3c82bba129172feb991d5af51afecd9d61a88b6832e4107480e392aed61a8644f551665ebff6b20953b635737a4f895e429fddcfe801f606fbda74b3bf6f5767d0fac14907fcfd0aa1d4c11b9e91b01d68052399b51a29f1ae6acd965109977c14a555cbcbd21ad8cb9f8853506d4bc21c01e62d61d7b21be1b923be54914e6b0a7ca84dd11f1159193e1184568a6134a6bbadf5b4df986edcf2019390ae841cfaa44435e28ce877d3dae4177992fa5d4e5c005876dbe3d1e63bec7dcc0942762b48b1ecc6c1a918409a8a72812a1e245c0c67be6e729c2b49bc6ee4d24a8f63e78e75db45655c26a9a78aff36fcd67117f26b8f654dca664b9f0e30681874cb749e1a692720078856286c2560b0292cc837933423147569350955c9571bf8941ba128fd339cb4268f46b94bc6ee203eb7026813706ea51c4f24c91866fc23a724bf2501327e6ae89c29f8db315dc28d2c7c719514036367e018f4835f63fdecd71f9bdced7132b6c4f8b13c69a517026fcd3622d67cb632320d5e7308f78f4b7cea11f6291b137851dc6cd6366f2785c71c3f237f81a7658b2a8d512b61e0ad5a4710b7b124151689fcb2116063fbff7e9115fed7b93de834970b838e49f8f8ba5f1f874c354078b5810a55ae289a56da563f1da6cd80a3757d6073fa55e016e45ac6cec1f69d871c92fd0ae9670c74249045e6b464787f9504128736309fed205f8df4d90e332908581298d9c75a3fa36ab0c3c9272e62de53ab290c803d67b696fd615c260a47bffad16746f18ba1a10a061bacbea9369693b3c042eec36bed289d7d12e52bca8aa1c2dff88ca7816498d25626d0f1e106ebb0b4a12138e00f3df5b1c2f49d98b1756e69b641b7c6353d99dbff050f4d76842c6cf1c2a4b062fc8e6336fa689b7c9d5c6b4ab8c15a5c20e514ff070a602d85ae52fa7810c22f8eeffd34a095b93342144f7a98d024216b3d68ed7bea047517bfcd83ec83febd1ba0e5858e2bdc1d8b1f7b0f89e90ccc432a3f930cb8209462e64556c5054c56ca2a85f16b32eb83a10459d13516faa4d23302b7607b9bd38dab2239ac9e9440c314433fdfb3ceadab4b4f87415ed6f240e017221f3b5f7ac196cdf54957bec42fe6893994b46de3d27dc7fb58ca88feb5b9e79cf20053d12530ac524337b22a3629bea52f40b06d3e2128f32060f9105847daed81d35f20e2002817434659baff64494c5b5c7f9216bfda38412a0f70511159dc73bb6bae1f8eaa0ef08d99bcb31f94f6be12c29c83df45926430b366c99fca3270c15fc4056398fdf3135b7779e3066a006961d1ac0ad1c83179ce39e87a96b722ec23aabc065badf3e188347a360772ca6a447abac7e6a44f0d4632d52926332e44a0a86bff5ce699fd063bdda3ffd4c41b53ded49fecec67f40599b934e16e3fd1bc063ad7026f8d71bfd4cbaf56599586774723194b692036f1b6bb242e2ffb9c600b5215b412764599476ce475c9e5b396fbcebd6be323dcf4d0048077400aac7500db41dc95fc7f7edbe7c9c2ec5ea89943fe13b42217eef530bbd023671509e12dfce4e1c1c82955d965e6a68aa66f6967dba48feda572db1f099d9a6dc4bc8edade852b5e824a06890dc48a6a6510ecaf8cf7620d757290e3166d431abecc624fa9ac2234d2eb783308ead45544910c633a94964b2ef5fbc409cb8835ac4147d384e12e0a5e13951f7de0ee13eafcb0ca0c04946d7804040c0a3cd088352424b097adb7aad1ca4495952f3e6c0158c02d2bcec33bfda69301434a84d9027ce02c0b9725dad118", "d894b86261436362e64241e61f6b3e6589daf64dc641f60570c4c0bf3b1f2ca3"); } +static MuHash3072 FromInt(unsigned char i) { + unsigned char tmp[32] = {i, 0}; + return MuHash3072(tmp); +} + +BOOST_AUTO_TEST_CASE(muhash_tests) +{ + uint256 out; + + for (int iter = 0; iter < 10; ++iter) { + uint256 res; + int table[4]; + for (int i = 0; i < 4; ++i) { + table[i] = g_insecure_rand_ctx.randbits(3); + } + for (int order = 0; order < 4; ++order) { + MuHash3072 acc; + for (int i = 0; i < 4; ++i) { + int t = table[i ^ order]; + if (t & 4) { + acc /= FromInt(t & 3); + } else { + acc *= FromInt(t & 3); + } + } + acc.Finalize(out); + if (order == 0) { + res = out; + } else { + BOOST_CHECK(res == out); + } + } + + MuHash3072 x = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X + MuHash3072 y = FromInt(g_insecure_rand_ctx.randbits(4)); // x=X, y=Y + MuHash3072 z; // x=X, y=Y, z=1 + z *= x; // x=X, y=Y, z=X + z *= y; // x=X, y=Y, z=X*Y + y *= x; // x=X, y=Y*X, z=X*Y + z /= y; // x=X, y=Y*X, z=1 + z.Finalize(out); + + uint256 out2; + MuHash3072 a; + a.Finalize(out2); + + BOOST_CHECK_EQUAL(out, out2); + } + + MuHash3072 acc = FromInt(0); + acc *= FromInt(1); + acc /= FromInt(2); + acc.Finalize(out); + BOOST_CHECK_EQUAL(out, uint256S("10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863")); + + MuHash3072 acc2 = FromInt(0); + unsigned char tmp[32] = {1, 0}; + acc2.Insert(tmp); + unsigned char tmp2[32] = {2, 0}; + acc2.Remove(tmp2); + acc2.Finalize(out); + BOOST_CHECK_EQUAL(out, uint256S("10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863")); + + // Test MuHash3072 serialization + MuHash3072 serchk = FromInt(1); serchk *= FromInt(2); + std::string ser_exp = "1fa093295ea30a6a3acdc7b3f770fa538eff537528e990e2910e40bbcfd7f6696b1256901929094694b56316de342f593303dd12ac43e06dce1be1ff8301c845beb15468fff0ef002dbf80c29f26e6452bccc91b5cb9437ad410d2a67ea847887fa3c6a6553309946880fe20db2c73fe0641adbd4e86edfee0d9f8cd0ee1230898873dc13ed8ddcaf045c80faa082774279007a2253f8922ee3ef361d378a6af3ddaf180b190ac97e556888c36b3d1fb1c85aab9ccd46e3deaeb7b7cf5db067a7e9ff86b658cf3acd6662bbcce37232daa753c48b794356c020090c831a8304416e2aa7ad633c0ddb2f11be1be316a81be7f7e472071c042cb68faef549c221ebff209273638b741aba5a81675c45a5fa92fea4ca821d7a324cb1e1a2ccd3b76c4228ec8066dad2a5df6e1bd0de45c7dd5de8070bdb46db6c554cf9aefc9b7b2bbf9f75b1864d9f95005314593905c0109b71f703d49944ae94477b51dac10a816bb6d1c700bafabc8bd86fac8df24be519a2f2836b16392e18036cb13e48c5c010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; + CDataStream ss_chk(SER_DISK, PROTOCOL_VERSION); + ss_chk << serchk; + BOOST_CHECK_EQUAL(ser_exp, HexStr(ss_chk.str())); + + // Test MuHash3072 deserialization + MuHash3072 deserchk; + ss_chk >> deserchk; + uint256 out3; + serchk.Finalize(out); + deserchk.Finalize(out3); + BOOST_CHECK_EQUAL(HexStr(out), HexStr(out3)); + + // Test MuHash3072 overflow, meaning the internal data is larger than the modulus. + CDataStream ss_max(ParseHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), SER_DISK, PROTOCOL_VERSION); + MuHash3072 overflowchk; + ss_max >> overflowchk; + + uint256 out4; + overflowchk.Finalize(out4); + BOOST_CHECK_EQUAL(HexStr(out4), "3a31e6903aff0de9f62f9a9f7f8b861de76ce2cda09822b90014319ae5dc2271"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index c399da900f..cf6009d591 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -80,11 +80,12 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = PeerManager::make(chainparams, *connman, nullptr, *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY); dummyNode1.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); @@ -135,7 +136,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerManager &peerLogic, CConnmanTest* connman) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); - vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY)); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY)); CNode &node = *vNodes.back(); node.SetCommonVersion(PROTOCOL_VERSION); @@ -149,7 +150,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = PeerManager::make(chainparams, *connman, nullptr, *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -222,11 +224,12 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = PeerManager::make(chainparams, *connman, banman.get(), *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode1(id++, NODE_NETWORK, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::INBOUND); dummyNode1.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); dummyNode1.fSuccessfullyConnected = true; @@ -239,7 +242,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode2(id++, NODE_NETWORK, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", ConnectionType::INBOUND); dummyNode2.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode2); dummyNode2.fSuccessfullyConnected = true; @@ -268,14 +271,15 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = PeerManager::make(chainparams, *connman, banman.get(), *m_node.scheduler, + *m_node.chainman, *m_node.mempool, false); banman->ClearBanned(); int64_t nStartTime = GetTime(); SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", ConnectionType::INBOUND); + CNode dummyNode(id++, NODE_NETWORK, INVALID_SOCKET, addr, 4, 4, CAddress(), "", ConnectionType::INBOUND); dummyNode.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode); dummyNode.fSuccessfullyConnected = true; diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index be7484cd0b..0c5c19113d 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 3e069eba69..744a9d78ce 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -14,6 +14,7 @@ #define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ #include <algorithm> +#include <array> #include <climits> #include <cstddef> #include <cstdint> @@ -34,272 +35,362 @@ class FuzzedDataProvider { : data_ptr_(data), remaining_bytes_(size) {} ~FuzzedDataProvider() = default; - // Returns a std::vector containing |num_bytes| of input data. If fewer than - // |num_bytes| of data remain, returns a shorter std::vector containing all - // of the data that's left. Can be used with any byte sized type, such as - // char, unsigned char, uint8_t, etc. - template <typename T> std::vector<T> ConsumeBytes(size_t num_bytes) { - num_bytes = std::min(num_bytes, remaining_bytes_); - return ConsumeBytes<T>(num_bytes, num_bytes); - } + // See the implementation below (after the class definition) for more verbose + // comments for each of the methods. - // Similar to |ConsumeBytes|, but also appends the terminator value at the end - // of the resulting vector. Useful, when a mutable null-terminated C-string is - // needed, for example. But that is a rare case. Better avoid it, if possible, - // and prefer using |ConsumeBytes| or |ConsumeBytesAsString| methods. + // Methods returning std::vector of bytes. These are the most popular choice + // when splitting fuzzing input into pieces, as every piece is put into a + // separate buffer (i.e. ASan would catch any under-/overflow) and the memory + // will be released automatically. + template <typename T> std::vector<T> ConsumeBytes(size_t num_bytes); template <typename T> - std::vector<T> ConsumeBytesWithTerminator(size_t num_bytes, - T terminator = 0) { - num_bytes = std::min(num_bytes, remaining_bytes_); - std::vector<T> result = ConsumeBytes<T>(num_bytes + 1, num_bytes); - result.back() = terminator; - return result; - } + std::vector<T> ConsumeBytesWithTerminator(size_t num_bytes, T terminator = 0); + template <typename T> std::vector<T> ConsumeRemainingBytes(); - // Returns a std::string containing |num_bytes| of input data. Using this and - // |.c_str()| on the resulting string is the best way to get an immutable - // null-terminated C string. If fewer than |num_bytes| of data remain, returns - // a shorter std::string containing all of the data that's left. - std::string ConsumeBytesAsString(size_t num_bytes) { - static_assert(sizeof(std::string::value_type) == sizeof(uint8_t), - "ConsumeBytesAsString cannot convert the data to a string."); - - num_bytes = std::min(num_bytes, remaining_bytes_); - std::string result( - reinterpret_cast<const std::string::value_type *>(data_ptr_), - num_bytes); - Advance(num_bytes); - return result; - } + // Methods returning strings. Use only when you need a std::string or a null + // terminated C-string. Otherwise, prefer the methods returning std::vector. + std::string ConsumeBytesAsString(size_t num_bytes); + std::string ConsumeRandomLengthString(size_t max_length); + std::string ConsumeRandomLengthString(); + std::string ConsumeRemainingBytesAsString(); - // Returns a number in the range [min, max] by consuming bytes from the - // input data. The value might not be uniformly distributed in the given - // range. If there's no input data left, always returns |min|. |min| must - // be less than or equal to |max|. - template <typename T> T ConsumeIntegralInRange(T min, T max) { - static_assert(std::is_integral<T>::value, "An integral type is required."); - static_assert(sizeof(T) <= sizeof(uint64_t), "Unsupported integral type."); + // Methods returning integer values. + template <typename T> T ConsumeIntegral(); + template <typename T> T ConsumeIntegralInRange(T min, T max); - if (min > max) - abort(); + // Methods returning floating point values. + template <typename T> T ConsumeFloatingPoint(); + template <typename T> T ConsumeFloatingPointInRange(T min, T max); - // Use the biggest type possible to hold the range and the result. - uint64_t range = static_cast<uint64_t>(max) - min; - uint64_t result = 0; - size_t offset = 0; - - while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 && - remaining_bytes_ != 0) { - // Pull bytes off the end of the seed data. Experimentally, this seems to - // allow the fuzzer to more easily explore the input space. This makes - // sense, since it works by modifying inputs that caused new code to run, - // and this data is often used to encode length of data read by - // |ConsumeBytes|. Separating out read lengths makes it easier modify the - // contents of the data that is actually read. - --remaining_bytes_; - result = (result << CHAR_BIT) | data_ptr_[remaining_bytes_]; - offset += CHAR_BIT; - } + // 0 <= return value <= 1. + template <typename T> T ConsumeProbability(); - // Avoid division by 0, in case |range + 1| results in overflow. - if (range != std::numeric_limits<decltype(range)>::max()) - result = result % (range + 1); + bool ConsumeBool(); - return static_cast<T>(min + result); - } + // Returns a value chosen from the given enum. + template <typename T> T ConsumeEnum(); - // Returns a std::string of length from 0 to |max_length|. When it runs out of - // input data, returns what remains of the input. Designed to be more stable - // with respect to a fuzzer inserting characters than just picking a random - // length and then consuming that many bytes with |ConsumeBytes|. - std::string ConsumeRandomLengthString(size_t max_length) { - // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\" - // followed by anything else to the end of the string. As a result of this - // logic, a fuzzer can insert characters into the string, and the string - // will be lengthened to include those new characters, resulting in a more - // stable fuzzer than picking the length of a string independently from - // picking its contents. - std::string result; - - // Reserve the anticipated capaticity to prevent several reallocations. - result.reserve(std::min(max_length, remaining_bytes_)); - for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) { - char next = ConvertUnsignedToSigned<char>(data_ptr_[0]); - Advance(1); - if (next == '\\' && remaining_bytes_ != 0) { - next = ConvertUnsignedToSigned<char>(data_ptr_[0]); - Advance(1); - if (next != '\\') - break; - } - result += next; - } - - result.shrink_to_fit(); - return result; - } + // Returns a value from the given array. + template <typename T, size_t size> T PickValueInArray(const T (&array)[size]); + template <typename T, size_t size> + T PickValueInArray(const std::array<T, size> &array); + template <typename T> T PickValueInArray(std::initializer_list<const T> list); - // Returns a std::vector containing all remaining bytes of the input data. - template <typename T> std::vector<T> ConsumeRemainingBytes() { - return ConsumeBytes<T>(remaining_bytes_); - } + // Writes data to the given destination and returns number of bytes written. + size_t ConsumeData(void *destination, size_t num_bytes); - // Returns a std::string containing all remaining bytes of the input data. - // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string - // object. - std::string ConsumeRemainingBytesAsString() { - return ConsumeBytesAsString(remaining_bytes_); - } + // Reports the remaining bytes available for fuzzed input. + size_t remaining_bytes() { return remaining_bytes_; } - // Returns a number in the range [Type's min, Type's max]. The value might - // not be uniformly distributed in the given range. If there's no input data - // left, always returns |min|. - template <typename T> T ConsumeIntegral() { - return ConsumeIntegralInRange(std::numeric_limits<T>::min(), - std::numeric_limits<T>::max()); - } + private: + FuzzedDataProvider(const FuzzedDataProvider &) = delete; + FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; - // Reads one byte and returns a bool, or false when no data remains. - bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); } + void CopyAndAdvance(void *destination, size_t num_bytes); - // Returns a copy of the value selected from the given fixed-size |array|. - template <typename T, size_t size> - T PickValueInArray(const T (&array)[size]) { - static_assert(size > 0, "The array must be non empty."); - return array[ConsumeIntegralInRange<size_t>(0, size - 1)]; - } + void Advance(size_t num_bytes); template <typename T> - T PickValueInArray(std::initializer_list<const T> list) { - // TODO(Dor1s): switch to static_assert once C++14 is allowed. - if (!list.size()) - abort(); - - return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); - } - - // Returns an enum value. The enum must start at 0 and be contiguous. It must - // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: - // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; - template <typename T> T ConsumeEnum() { - static_assert(std::is_enum<T>::value, "|T| must be an enum type."); - return static_cast<T>(ConsumeIntegralInRange<uint32_t>( - 0, static_cast<uint32_t>(T::kMaxValue))); - } + std::vector<T> ConsumeBytes(size_t size, size_t num_bytes); - // Returns a floating point number in the range [0.0, 1.0]. If there's no - // input data left, always returns 0. - template <typename T> T ConsumeProbability() { - static_assert(std::is_floating_point<T>::value, - "A floating point type is required."); + template <typename TS, typename TU> TS ConvertUnsignedToSigned(TU value); - // Use different integral types for different floating point types in order - // to provide better density of the resulting values. - using IntegralType = - typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t, - uint64_t>::type; + const uint8_t *data_ptr_; + size_t remaining_bytes_; +}; - T result = static_cast<T>(ConsumeIntegral<IntegralType>()); - result /= static_cast<T>(std::numeric_limits<IntegralType>::max()); - return result; +// Returns a std::vector containing |num_bytes| of input data. If fewer than +// |num_bytes| of data remain, returns a shorter std::vector containing all +// of the data that's left. Can be used with any byte sized type, such as +// char, unsigned char, uint8_t, etc. +template <typename T> +std::vector<T> FuzzedDataProvider::ConsumeBytes(size_t num_bytes) { + num_bytes = std::min(num_bytes, remaining_bytes_); + return ConsumeBytes<T>(num_bytes, num_bytes); +} + +// Similar to |ConsumeBytes|, but also appends the terminator value at the end +// of the resulting vector. Useful, when a mutable null-terminated C-string is +// needed, for example. But that is a rare case. Better avoid it, if possible, +// and prefer using |ConsumeBytes| or |ConsumeBytesAsString| methods. +template <typename T> +std::vector<T> FuzzedDataProvider::ConsumeBytesWithTerminator(size_t num_bytes, + T terminator) { + num_bytes = std::min(num_bytes, remaining_bytes_); + std::vector<T> result = ConsumeBytes<T>(num_bytes + 1, num_bytes); + result.back() = terminator; + return result; +} + +// Returns a std::vector containing all remaining bytes of the input data. +template <typename T> +std::vector<T> FuzzedDataProvider::ConsumeRemainingBytes() { + return ConsumeBytes<T>(remaining_bytes_); +} + +// Returns a std::string containing |num_bytes| of input data. Using this and +// |.c_str()| on the resulting string is the best way to get an immutable +// null-terminated C string. If fewer than |num_bytes| of data remain, returns +// a shorter std::string containing all of the data that's left. +inline std::string FuzzedDataProvider::ConsumeBytesAsString(size_t num_bytes) { + static_assert(sizeof(std::string::value_type) == sizeof(uint8_t), + "ConsumeBytesAsString cannot convert the data to a string."); + + num_bytes = std::min(num_bytes, remaining_bytes_); + std::string result( + reinterpret_cast<const std::string::value_type *>(data_ptr_), num_bytes); + Advance(num_bytes); + return result; +} + +// Returns a std::string of length from 0 to |max_length|. When it runs out of +// input data, returns what remains of the input. Designed to be more stable +// with respect to a fuzzer inserting characters than just picking a random +// length and then consuming that many bytes with |ConsumeBytes|. +inline std::string +FuzzedDataProvider::ConsumeRandomLengthString(size_t max_length) { + // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\" + // followed by anything else to the end of the string. As a result of this + // logic, a fuzzer can insert characters into the string, and the string + // will be lengthened to include those new characters, resulting in a more + // stable fuzzer than picking the length of a string independently from + // picking its contents. + std::string result; + + // Reserve the anticipated capaticity to prevent several reallocations. + result.reserve(std::min(max_length, remaining_bytes_)); + for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) { + char next = ConvertUnsignedToSigned<char>(data_ptr_[0]); + Advance(1); + if (next == '\\' && remaining_bytes_ != 0) { + next = ConvertUnsignedToSigned<char>(data_ptr_[0]); + Advance(1); + if (next != '\\') + break; + } + result += next; } - // Returns a floating point value in the range [Type's lowest, Type's max] by - // consuming bytes from the input data. If there's no input data left, always - // returns approximately 0. - template <typename T> T ConsumeFloatingPoint() { - return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(), - std::numeric_limits<T>::max()); + result.shrink_to_fit(); + return result; +} + +// Returns a std::string of length from 0 to |remaining_bytes_|. +inline std::string FuzzedDataProvider::ConsumeRandomLengthString() { + return ConsumeRandomLengthString(remaining_bytes_); +} + +// Returns a std::string containing all remaining bytes of the input data. +// Prefer using |ConsumeRemainingBytes| unless you actually need a std::string +// object. +inline std::string FuzzedDataProvider::ConsumeRemainingBytesAsString() { + return ConsumeBytesAsString(remaining_bytes_); +} + +// Returns a number in the range [Type's min, Type's max]. The value might +// not be uniformly distributed in the given range. If there's no input data +// left, always returns |min|. +template <typename T> T FuzzedDataProvider::ConsumeIntegral() { + return ConsumeIntegralInRange(std::numeric_limits<T>::min(), + std::numeric_limits<T>::max()); +} + +// Returns a number in the range [min, max] by consuming bytes from the +// input data. The value might not be uniformly distributed in the given +// range. If there's no input data left, always returns |min|. |min| must +// be less than or equal to |max|. +template <typename T> +T FuzzedDataProvider::ConsumeIntegralInRange(T min, T max) { + static_assert(std::is_integral<T>::value, "An integral type is required."); + static_assert(sizeof(T) <= sizeof(uint64_t), "Unsupported integral type."); + + if (min > max) + abort(); + + // Use the biggest type possible to hold the range and the result. + uint64_t range = static_cast<uint64_t>(max) - min; + uint64_t result = 0; + size_t offset = 0; + + while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 && + remaining_bytes_ != 0) { + // Pull bytes off the end of the seed data. Experimentally, this seems to + // allow the fuzzer to more easily explore the input space. This makes + // sense, since it works by modifying inputs that caused new code to run, + // and this data is often used to encode length of data read by + // |ConsumeBytes|. Separating out read lengths makes it easier modify the + // contents of the data that is actually read. + --remaining_bytes_; + result = (result << CHAR_BIT) | data_ptr_[remaining_bytes_]; + offset += CHAR_BIT; } - // Returns a floating point value in the given range by consuming bytes from - // the input data. If there's no input data left, returns |min|. Note that - // |min| must be less than or equal to |max|. - template <typename T> T ConsumeFloatingPointInRange(T min, T max) { - if (min > max) - abort(); - - T range = .0; - T result = min; - constexpr T zero(.0); - if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) { - // The diff |max - min| would overflow the given floating point type. Use - // the half of the diff as the range and consume a bool to decide whether - // the result is in the first of the second part of the diff. - range = (max / 2.0) - (min / 2.0); - if (ConsumeBool()) { - result += range; - } - } else { - range = max - min; + // Avoid division by 0, in case |range + 1| results in overflow. + if (range != std::numeric_limits<decltype(range)>::max()) + result = result % (range + 1); + + return static_cast<T>(min + result); +} + +// Returns a floating point value in the range [Type's lowest, Type's max] by +// consuming bytes from the input data. If there's no input data left, always +// returns approximately 0. +template <typename T> T FuzzedDataProvider::ConsumeFloatingPoint() { + return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(), + std::numeric_limits<T>::max()); +} + +// Returns a floating point value in the given range by consuming bytes from +// the input data. If there's no input data left, returns |min|. Note that +// |min| must be less than or equal to |max|. +template <typename T> +T FuzzedDataProvider::ConsumeFloatingPointInRange(T min, T max) { + if (min > max) + abort(); + + T range = .0; + T result = min; + constexpr T zero(.0); + if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) { + // The diff |max - min| would overflow the given floating point type. Use + // the half of the diff as the range and consume a bool to decide whether + // the result is in the first of the second part of the diff. + range = (max / 2.0) - (min / 2.0); + if (ConsumeBool()) { + result += range; } - - return result + range * ConsumeProbability<T>(); + } else { + range = max - min; } - // Reports the remaining bytes available for fuzzed input. - size_t remaining_bytes() { return remaining_bytes_; } - - private: - FuzzedDataProvider(const FuzzedDataProvider &) = delete; - FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; - - void Advance(size_t num_bytes) { - if (num_bytes > remaining_bytes_) + return result + range * ConsumeProbability<T>(); +} + +// Returns a floating point number in the range [0.0, 1.0]. If there's no +// input data left, always returns 0. +template <typename T> T FuzzedDataProvider::ConsumeProbability() { + static_assert(std::is_floating_point<T>::value, + "A floating point type is required."); + + // Use different integral types for different floating point types in order + // to provide better density of the resulting values. + using IntegralType = + typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t, + uint64_t>::type; + + T result = static_cast<T>(ConsumeIntegral<IntegralType>()); + result /= static_cast<T>(std::numeric_limits<IntegralType>::max()); + return result; +} + +// Reads one byte and returns a bool, or false when no data remains. +inline bool FuzzedDataProvider::ConsumeBool() { + return 1 & ConsumeIntegral<uint8_t>(); +} + +// Returns an enum value. The enum must start at 0 and be contiguous. It must +// also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: +// enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; +template <typename T> T FuzzedDataProvider::ConsumeEnum() { + static_assert(std::is_enum<T>::value, "|T| must be an enum type."); + return static_cast<T>( + ConsumeIntegralInRange<uint32_t>(0, static_cast<uint32_t>(T::kMaxValue))); +} + +// Returns a copy of the value selected from the given fixed-size |array|. +template <typename T, size_t size> +T FuzzedDataProvider::PickValueInArray(const T (&array)[size]) { + static_assert(size > 0, "The array must be non empty."); + return array[ConsumeIntegralInRange<size_t>(0, size - 1)]; +} + +template <typename T, size_t size> +T FuzzedDataProvider::PickValueInArray(const std::array<T, size> &array) { + static_assert(size > 0, "The array must be non empty."); + return array[ConsumeIntegralInRange<size_t>(0, size - 1)]; +} + +template <typename T> +T FuzzedDataProvider::PickValueInArray(std::initializer_list<const T> list) { + // TODO(Dor1s): switch to static_assert once C++14 is allowed. + if (!list.size()) + abort(); + + return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); +} + +// Writes |num_bytes| of input data to the given destination pointer. If there +// is not enough data left, writes all remaining bytes. Return value is the +// number of bytes written. +// In general, it's better to avoid using this function, but it may be useful +// in cases when it's necessary to fill a certain buffer or object with +// fuzzing data. +inline size_t FuzzedDataProvider::ConsumeData(void *destination, + size_t num_bytes) { + num_bytes = std::min(num_bytes, remaining_bytes_); + CopyAndAdvance(destination, num_bytes); + return num_bytes; +} + +// Private methods. +inline void FuzzedDataProvider::CopyAndAdvance(void *destination, + size_t num_bytes) { + std::memcpy(destination, data_ptr_, num_bytes); + Advance(num_bytes); +} + +inline void FuzzedDataProvider::Advance(size_t num_bytes) { + if (num_bytes > remaining_bytes_) + abort(); + + data_ptr_ += num_bytes; + remaining_bytes_ -= num_bytes; +} + +template <typename T> +std::vector<T> FuzzedDataProvider::ConsumeBytes(size_t size, size_t num_bytes) { + static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type."); + + // The point of using the size-based constructor below is to increase the + // odds of having a vector object with capacity being equal to the length. + // That part is always implementation specific, but at least both libc++ and + // libstdc++ allocate the requested number of bytes in that constructor, + // which seems to be a natural choice for other implementations as well. + // To increase the odds even more, we also call |shrink_to_fit| below. + std::vector<T> result(size); + if (size == 0) { + if (num_bytes != 0) abort(); - - data_ptr_ += num_bytes; - remaining_bytes_ -= num_bytes; - } - - template <typename T> - std::vector<T> ConsumeBytes(size_t size, size_t num_bytes_to_consume) { - static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type."); - - // The point of using the size-based constructor below is to increase the - // odds of having a vector object with capacity being equal to the length. - // That part is always implementation specific, but at least both libc++ and - // libstdc++ allocate the requested number of bytes in that constructor, - // which seems to be a natural choice for other implementations as well. - // To increase the odds even more, we also call |shrink_to_fit| below. - std::vector<T> result(size); - if (size == 0) { - if (num_bytes_to_consume != 0) - abort(); - return result; - } - - std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); - Advance(num_bytes_to_consume); - - // Even though |shrink_to_fit| is also implementation specific, we expect it - // to provide an additional assurance in case vector's constructor allocated - // a buffer which is larger than the actual amount of data we put inside it. - result.shrink_to_fit(); return result; } - template <typename TS, typename TU> TS ConvertUnsignedToSigned(TU value) { - static_assert(sizeof(TS) == sizeof(TU), "Incompatible data types."); - static_assert(!std::numeric_limits<TU>::is_signed, - "Source type must be unsigned."); - - // TODO(Dor1s): change to `if constexpr` once C++17 becomes mainstream. - if (std::numeric_limits<TS>::is_modulo) - return static_cast<TS>(value); - - // Avoid using implementation-defined unsigned to signer conversions. - // To learn more, see https://stackoverflow.com/questions/13150449. - if (value <= std::numeric_limits<TS>::max()) { - return static_cast<TS>(value); - } else { - constexpr auto TS_min = std::numeric_limits<TS>::min(); - return TS_min + static_cast<char>(value - TS_min); - } + CopyAndAdvance(result.data(), num_bytes); + + // Even though |shrink_to_fit| is also implementation specific, we expect it + // to provide an additional assurance in case vector's constructor allocated + // a buffer which is larger than the actual amount of data we put inside it. + result.shrink_to_fit(); + return result; +} + +template <typename TS, typename TU> +TS FuzzedDataProvider::ConvertUnsignedToSigned(TU value) { + static_assert(sizeof(TS) == sizeof(TU), "Incompatible data types."); + static_assert(!std::numeric_limits<TU>::is_signed, + "Source type must be unsigned."); + + // TODO(Dor1s): change to `if constexpr` once C++17 becomes mainstream. + if (std::numeric_limits<TS>::is_modulo) + return static_cast<TS>(value); + + // Avoid using implementation-defined unsigned to signed conversions. + // To learn more, see https://stackoverflow.com/questions/13150449. + if (value <= std::numeric_limits<TS>::max()) { + return static_cast<TS>(value); + } else { + constexpr auto TS_min = std::numeric_limits<TS>::min(); + return TS_min + static_cast<char>(value - TS_min); } - - const uint8_t *data_ptr_; - size_t remaining_bytes_; -}; +} #endif // LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp index a455992b13..c6cfbd8d30 100644 --- a/src/test/fuzz/addition_overflow.cpp +++ b/src/test/fuzz/addition_overflow.cpp @@ -14,7 +14,7 @@ #if __has_builtin(__builtin_add_overflow) #define HAVE_BUILTIN_ADD_OVERFLOW #endif -#elif defined(__GNUC__) && (__GNUC__ >= 5) +#elif defined(__GNUC__) #define HAVE_BUILTIN_ADD_OVERFLOW #endif @@ -40,7 +40,7 @@ void TestAdditionOverflow(FuzzedDataProvider& fuzzed_data_provider) } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(addition_overflow) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); TestAdditionOverflow<int64_t>(fuzzed_data_provider); diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index 16b1cb755a..d15c785673 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(addrdb) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 0ceeea2d36..af9080b5e9 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -17,17 +17,27 @@ #include <string> #include <vector> -void initialize() +void initialize_addrman() { SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +class CAddrManDeterministic : public CAddrMan { - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); +public: + void MakeDeterministic(const uint256& random_seed) + { + insecure_rand = FastRandomContext{random_seed}; + Clear(); + } +}; +FUZZ_TARGET_INIT(addrman, initialize_addrman) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - CAddrMan addr_man; + CAddrManDeterministic addr_man; + addr_man.MakeDeterministic(ConsumeUInt256(fuzzed_data_provider)); if (fuzzed_data_provider.ConsumeBool()) { addr_man.m_asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); if (!SanityCheckASMap(addr_man.m_asmap)) { diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index e3aefa18a3..4c5bc0cbf2 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -27,7 +27,7 @@ static const std::vector<bool> IPV4_PREFIX_ASMAP = { true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true // Match 0xFF }; -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(asmap) { // Encoding: [7 bits: asmap size] [1 bit: ipv6?] [3-130 bytes: asmap] [4 or 16 bytes: addr] if (buffer.size() < 1 + 3 + 4) return; diff --git a/src/test/fuzz/asmap_direct.cpp b/src/test/fuzz/asmap_direct.cpp index 2d21eff9d6..8b7822dc16 100644 --- a/src/test/fuzz/asmap_direct.cpp +++ b/src/test/fuzz/asmap_direct.cpp @@ -11,7 +11,7 @@ #include <assert.h> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(asmap_direct) { // Encoding: [asmap using 1 bit / byte] 0xFF [addr using 1 bit / byte] std::optional<size_t> sep_pos_opt; diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 7ea0bdd2a7..eb3424ef28 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -15,7 +15,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index fc4a1d9261..cf69fa0722 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -24,14 +24,15 @@ int64_t ConsumeBanTimeOffset(FuzzedDataProvider& fuzzed_data_provider) noexcept } } // namespace -void initialize() +void initialize_banman() { InitializeFuzzingContext(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(banman, initialize_banman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); const fs::path banlist_file = GetDataDir() / "fuzzed_banlist.dat"; fs::remove(banlist_file); { diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index 8d49f93c2f..4470e13a61 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -14,7 +14,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(base_encode_decode) { const std::string random_encoded_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/bech32.cpp b/src/test/fuzz/bech32.cpp index 8b91f9bc96..95cd4b413f 100644 --- a/src/test/fuzz/bech32.cpp +++ b/src/test/fuzz/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,7 @@ #include <utility> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(bech32) { const std::string random_string(buffer.begin(), buffer.end()); const std::pair<std::string, std::vector<uint8_t>> r1 = bech32::Decode(random_string); diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 91bd34a251..65a33de4b4 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -17,13 +17,13 @@ #include <cassert> #include <string> -void initialize() +void initialize_block() { static const ECCVerifyHandle verify_handle; SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(block, initialize_block) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); CBlock block; diff --git a/src/test/fuzz/block_header.cpp b/src/test/fuzz/block_header.cpp index 09c2b4a951..c73270dcb3 100644 --- a/src/test/fuzz/block_header.cpp +++ b/src/test/fuzz/block_header.cpp @@ -14,7 +14,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(block_header) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); diff --git a/src/test/fuzz/blockfilter.cpp b/src/test/fuzz/blockfilter.cpp index 7232325a20..7fa06085f8 100644 --- a/src/test/fuzz/blockfilter.cpp +++ b/src/test/fuzz/blockfilter.cpp @@ -12,7 +12,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(blockfilter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index d955c71bc9..c0c66c564b 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -15,7 +15,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index e575640be5..23e197456a 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -15,7 +15,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(buffered_file) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index 47c71850ce..9f7074b423 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -11,7 +11,7 @@ #include <optional> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(chain) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); std::optional<CDiskBlockIndex> disk_block_index = ConsumeDeserializable<CDiskBlockIndex>(fuzzed_data_provider); diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index c69043bb6b..0b16f0f0d5 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -32,7 +32,7 @@ struct DumbCheck { }; } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(checkqueue) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index ac034809b0..1ae421493e 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -34,14 +34,14 @@ bool operator==(const Coin& a, const Coin& b) } } // namespace -void initialize() +void initialize_coins_view() { static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(coins_view, initialize_coins_view) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CCoinsView backend_coins_view; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 6521c3f3b2..fdf51d8558 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -15,14 +15,15 @@ #include <cstdint> #include <vector> -void initialize() +void initialize_connman() { InitializeFuzzingContext(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeBool()}; CAddress random_address; CNetAddr random_netaddr; @@ -31,7 +32,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CSubNet random_subnet; std::string random_string; while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 30)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 27)) { case 0: random_address = ConsumeAddress(fuzzed_data_provider); break; @@ -124,28 +125,18 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 25: - connman.SetBestHeight(fuzzed_data_provider.ConsumeIntegral<int>()); + connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); break; case 26: - connman.SetMaxOutboundTarget(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + connman.SetServices(random_service, ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS)); break; case 27: - connman.SetMaxOutboundTimeframe(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - break; - case 28: - connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); - break; - case 29: - connman.SetServices(random_service, static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>())); - break; - case 30: connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); break; } } (void)connman.GetAddedNodeInfo(); - (void)connman.GetBestHeight(); - (void)connman.GetExtraOutboundCount(); + (void)connman.GetExtraFullOutboundCount(); (void)connman.GetLocalServices(); (void)connman.GetMaxOutboundTarget(); (void)connman.GetMaxOutboundTimeframe(); diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp index 664e65accc..9668b84e7b 100644 --- a/src/test/fuzz/crypto.cpp +++ b/src/test/fuzz/crypto.cpp @@ -4,6 +4,7 @@ #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> +#include <crypto/muhash.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -17,7 +18,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); @@ -35,6 +36,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CSHA512 sha512; SHA3_256 sha3; CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; + MuHash3072 muhash; while (fuzzed_data_provider.ConsumeBool()) { switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { @@ -60,6 +62,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)Hash(data); (void)Hash160(data); (void)sha512.Size(); + + if (fuzzed_data_provider.ConsumeBool()) { + muhash *= MuHash3072(data); + } else { + muhash /= MuHash3072(data); + } break; } case 1: { @@ -70,10 +78,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)sha256.Reset(); (void)sha3.Reset(); (void)sha512.Reset(); + muhash = MuHash3072(); break; } case 2: { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 9)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)) { case 0: { data.resize(CHash160::OUTPUT_SIZE); hash160.Finalize(data); @@ -124,6 +133,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) sha3.Finalize(data); break; } + case 10: { + uint256 out; + muhash.Finalize(out); + break; + } } break; } diff --git a/src/test/fuzz/crypto_aes256.cpp b/src/test/fuzz/crypto_aes256.cpp index ae14073c96..ccabd1f7dc 100644 --- a/src/test/fuzz/crypto_aes256.cpp +++ b/src/test/fuzz/crypto_aes256.cpp @@ -11,7 +11,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_aes256) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); diff --git a/src/test/fuzz/crypto_aes256cbc.cpp b/src/test/fuzz/crypto_aes256cbc.cpp index 52983c7e79..6d4138e546 100644 --- a/src/test/fuzz/crypto_aes256cbc.cpp +++ b/src/test/fuzz/crypto_aes256cbc.cpp @@ -11,7 +11,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_aes256cbc) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES256_KEYSIZE); diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index b7438d312d..d751466f11 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -10,7 +10,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_chacha20) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index 48e4263f27..631af9c70d 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -13,7 +13,7 @@ #include <limits> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_chacha20_poly1305_aead) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; diff --git a/src/test/fuzz/crypto_common.cpp b/src/test/fuzz/crypto_common.cpp index 7ccb125216..8e07dfedb9 100644 --- a/src/test/fuzz/crypto_common.cpp +++ b/src/test/fuzz/crypto_common.cpp @@ -13,7 +13,7 @@ #include <cstring> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_common) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const uint16_t random_u16 = fuzzed_data_provider.ConsumeIntegral<uint16_t>(); diff --git a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp index e0a4e90c10..8cb9c55283 100644 --- a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp +++ b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp @@ -11,7 +11,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_hkdf_hmac_sha256_l32) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; diff --git a/src/test/fuzz/crypto_poly1305.cpp b/src/test/fuzz/crypto_poly1305.cpp index 5681e6a693..ac555ed68c 100644 --- a/src/test/fuzz/crypto_poly1305.cpp +++ b/src/test/fuzz/crypto_poly1305.cpp @@ -10,7 +10,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(crypto_poly1305) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp index 5b45aa79d8..dc20dc3f62 100644 --- a/src/test/fuzz/cuckoocache.cpp +++ b/src/test/fuzz/cuckoocache.cpp @@ -26,7 +26,7 @@ struct RandomHasher { }; } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(cuckoocache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); fuzzed_data_provider_ptr = &fuzzed_data_provider; diff --git a/src/test/fuzz/danger_link_all.sh b/src/test/fuzz/danger_link_all.sh new file mode 100755 index 0000000000..2ddd00c658 --- /dev/null +++ b/src/test/fuzz/danger_link_all.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -e + +ROOT_DIR="$(git rev-parse --show-toplevel)" + +# Run only once (break make recursion) +if [ -d "${ROOT_DIR}/lock_fuzz_link_all" ]; then + exit +fi +mkdir "${ROOT_DIR}/lock_fuzz_link_all" + +echo "Linking each fuzz target separately." +for FUZZING_HARNESS in $(PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 "${ROOT_DIR}/src/test/fuzz/fuzz" | sort -u); do + echo "Building src/test/fuzz/${FUZZING_HARNESS} ..." + git checkout -- "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" + sed -i "s/std::getenv(\"FUZZ\")/\"${FUZZING_HARNESS}\"/g" "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" + make + mv "${ROOT_DIR}/src/test/fuzz/fuzz" "${ROOT_DIR}/src/test/fuzz/${FUZZING_HARNESS}" +done +git checkout -- "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" +rmdir "${ROOT_DIR}/lock_fuzz_link_all" +echo "Successfully built all fuzz targets." diff --git a/src/test/fuzz/data_stream.cpp b/src/test/fuzz/data_stream.cpp new file mode 100644 index 0000000000..28fc528ceb --- /dev/null +++ b/src/test/fuzz/data_stream.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrman.h> +#include <net.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +void initialize_data_stream_addr_man() +{ + InitializeFuzzingContext(); +} + +FUZZ_TARGET_INIT(data_stream_addr_man, initialize_data_stream_addr_man) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); + CAddrMan addr_man; + CAddrDB::Read(addr_man, data_stream); +} diff --git a/src/test/fuzz/decode_tx.cpp b/src/test/fuzz/decode_tx.cpp index a2b18c0365..57431e65ba 100644 --- a/src/test/fuzz/decode_tx.cpp +++ b/src/test/fuzz/decode_tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(decode_tx) { const std::string tx_hex = HexStr(buffer); CMutableTransaction mtx; diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 7b57a2c1e2..0d1921f285 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -8,14 +8,14 @@ #include <test/fuzz/fuzz.h> #include <util/memory.h> -void initialize() +void initialize_descriptor_parse() { static const ECCVerifyHandle verify_handle; ECC_Start(); SelectParams(CBaseChainParams::MAIN); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(descriptor_parse, initialize_descriptor_parse) { const std::string descriptor(buffer.begin(), buffer.end()); FlatSigningProvider signing_provider; diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 8ca5366c8a..74dec6475e 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -34,12 +34,21 @@ #include <test/fuzz/fuzz.h> -void initialize() +void initialize_deserialize() { // Fuzzers using pubkey must hold an ECCVerifyHandle. static const ECCVerifyHandle verify_handle; } +#define FUZZ_TARGET_DESERIALIZE(name, code) \ + FUZZ_TARGET_INIT(name, initialize_deserialize) \ + { \ + try { \ + code \ + } catch (const invalid_fuzzing_input_exception&) { \ + } \ + } + namespace { struct invalid_fuzzing_input_exception : public std::exception { @@ -92,44 +101,51 @@ void AssertEqualAfterSerializeDeserialize(const T& obj, const int version = INIT } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) -{ - try { -#if BLOCK_FILTER_DESERIALIZE +FUZZ_TARGET_DESERIALIZE(block_filter_deserialize, { BlockFilter block_filter; DeserializeFromFuzzingInput(buffer, block_filter); -#elif ADDR_INFO_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(addr_info_deserialize, { CAddrInfo addr_info; DeserializeFromFuzzingInput(buffer, addr_info); -#elif BLOCK_FILE_INFO_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(block_file_info_deserialize, { CBlockFileInfo block_file_info; DeserializeFromFuzzingInput(buffer, block_file_info); -#elif BLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(block_header_and_short_txids_deserialize, { CBlockHeaderAndShortTxIDs block_header_and_short_txids; DeserializeFromFuzzingInput(buffer, block_header_and_short_txids); -#elif FEE_RATE_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(fee_rate_deserialize, { CFeeRate fee_rate; DeserializeFromFuzzingInput(buffer, fee_rate); AssertEqualAfterSerializeDeserialize(fee_rate); -#elif MERKLE_BLOCK_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(merkle_block_deserialize, { CMerkleBlock merkle_block; DeserializeFromFuzzingInput(buffer, merkle_block); -#elif OUT_POINT_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(out_point_deserialize, { COutPoint out_point; DeserializeFromFuzzingInput(buffer, out_point); AssertEqualAfterSerializeDeserialize(out_point); -#elif PARTIAL_MERKLE_TREE_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(partial_merkle_tree_deserialize, { CPartialMerkleTree partial_merkle_tree; DeserializeFromFuzzingInput(buffer, partial_merkle_tree); -#elif PUB_KEY_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(pub_key_deserialize, { CPubKey pub_key; DeserializeFromFuzzingInput(buffer, pub_key); // TODO: The following equivalence should hold for CPubKey? Fix. // AssertEqualAfterSerializeDeserialize(pub_key); -#elif SCRIPT_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(script_deserialize, { CScript script; DeserializeFromFuzzingInput(buffer, script); -#elif SUB_NET_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(sub_net_deserialize, { CSubNet sub_net_1; DeserializeFromFuzzingInput(buffer, sub_net_1, INIT_PROTO_VERSION); AssertEqualAfterSerializeDeserialize(sub_net_1, INIT_PROTO_VERSION); @@ -139,67 +155,85 @@ void test_one_input(const std::vector<uint8_t>& buffer) CSubNet sub_net_3; DeserializeFromFuzzingInput(buffer, sub_net_3); AssertEqualAfterSerializeDeserialize(sub_net_3, INIT_PROTO_VERSION | ADDRV2_FORMAT); -#elif TX_IN_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(tx_in_deserialize, { CTxIn tx_in; DeserializeFromFuzzingInput(buffer, tx_in); AssertEqualAfterSerializeDeserialize(tx_in); -#elif FLAT_FILE_POS_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(flat_file_pos_deserialize, { FlatFilePos flat_file_pos; DeserializeFromFuzzingInput(buffer, flat_file_pos); AssertEqualAfterSerializeDeserialize(flat_file_pos); -#elif KEY_ORIGIN_INFO_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(key_origin_info_deserialize, { KeyOriginInfo key_origin_info; DeserializeFromFuzzingInput(buffer, key_origin_info); AssertEqualAfterSerializeDeserialize(key_origin_info); -#elif PARTIALLY_SIGNED_TRANSACTION_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(partially_signed_transaction_deserialize, { PartiallySignedTransaction partially_signed_transaction; DeserializeFromFuzzingInput(buffer, partially_signed_transaction); -#elif PREFILLED_TRANSACTION_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(prefilled_transaction_deserialize, { PrefilledTransaction prefilled_transaction; DeserializeFromFuzzingInput(buffer, prefilled_transaction); -#elif PSBT_INPUT_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(psbt_input_deserialize, { PSBTInput psbt_input; DeserializeFromFuzzingInput(buffer, psbt_input); -#elif PSBT_OUTPUT_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(psbt_output_deserialize, { PSBTOutput psbt_output; DeserializeFromFuzzingInput(buffer, psbt_output); -#elif BLOCK_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(block_deserialize, { CBlock block; DeserializeFromFuzzingInput(buffer, block); -#elif BLOCKLOCATOR_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { CBlockLocator bl; DeserializeFromFuzzingInput(buffer, bl); -#elif BLOCKMERKLEROOT +}) +FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { CBlock block; DeserializeFromFuzzingInput(buffer, block); bool mutated; BlockMerkleRoot(block, &mutated); -#elif ADDRMAN_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { CAddrMan am; DeserializeFromFuzzingInput(buffer, am); -#elif BLOCKHEADER_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { CBlockHeader bh; DeserializeFromFuzzingInput(buffer, bh); -#elif BANENTRY_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(banentry_deserialize, { CBanEntry be; DeserializeFromFuzzingInput(buffer, be); -#elif TXUNDO_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(txundo_deserialize, { CTxUndo tu; DeserializeFromFuzzingInput(buffer, tu); -#elif BLOCKUNDO_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(blockundo_deserialize, { CBlockUndo bu; DeserializeFromFuzzingInput(buffer, bu); -#elif COINS_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(coins_deserialize, { Coin coin; DeserializeFromFuzzingInput(buffer, coin); -#elif NETADDR_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(netaddr_deserialize, { CNetAddr na; DeserializeFromFuzzingInput(buffer, na); if (na.IsAddrV1Compatible()) { AssertEqualAfterSerializeDeserialize(na); } AssertEqualAfterSerializeDeserialize(na, INIT_PROTO_VERSION | ADDRV2_FORMAT); -#elif SERVICE_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(service_deserialize, { CService s; DeserializeFromFuzzingInput(buffer, s); if (s.IsAddrV1Compatible()) { @@ -213,50 +247,56 @@ void test_one_input(const std::vector<uint8_t>& buffer) CService s2; DeserializeFromFuzzingInput(buffer, s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); AssertEqualAfterSerializeDeserialize(s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); -#elif MESSAGEHEADER_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(messageheader_deserialize, { CMessageHeader mh; DeserializeFromFuzzingInput(buffer, mh); (void)mh.IsCommandValid(); -#elif ADDRESS_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(address_deserialize, { CAddress a; DeserializeFromFuzzingInput(buffer, a); -#elif INV_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(inv_deserialize, { CInv i; DeserializeFromFuzzingInput(buffer, i); -#elif BLOOMFILTER_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(bloomfilter_deserialize, { CBloomFilter bf; DeserializeFromFuzzingInput(buffer, bf); -#elif DISKBLOCKINDEX_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(diskblockindex_deserialize, { CDiskBlockIndex dbi; DeserializeFromFuzzingInput(buffer, dbi); -#elif TXOUTCOMPRESSOR_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(txoutcompressor_deserialize, { CTxOut to; auto toc = Using<TxOutCompression>(to); DeserializeFromFuzzingInput(buffer, toc); -#elif BLOCKTRANSACTIONS_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(blocktransactions_deserialize, { BlockTransactions bt; DeserializeFromFuzzingInput(buffer, bt); -#elif BLOCKTRANSACTIONSREQUEST_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(blocktransactionsrequest_deserialize, { BlockTransactionsRequest btr; DeserializeFromFuzzingInput(buffer, btr); -#elif SNAPSHOTMETADATA_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(snapshotmetadata_deserialize, { SnapshotMetadata snapshot_metadata; DeserializeFromFuzzingInput(buffer, snapshot_metadata); -#elif UINT160_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(uint160_deserialize, { uint160 u160; DeserializeFromFuzzingInput(buffer, u160); AssertEqualAfterSerializeDeserialize(u160); -#elif UINT256_DESERIALIZE +}) +FUZZ_TARGET_DESERIALIZE(uint256_deserialize, { uint256 u256; DeserializeFromFuzzingInput(buffer, u256); AssertEqualAfterSerializeDeserialize(u256); -#else -#error Need at least one fuzz target to compile -#endif +}) // Classes intentionally not covered in this file since their deserialization code is // fuzzed elsewhere: // * Deserialization of CTxOut is fuzzed in test/fuzz/tx_out.cpp // * Deserialization of CMutableTransaction is fuzzed in src/test/fuzz/transaction.cpp - } catch (const invalid_fuzzing_input_exception&) { - } -} diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index c556599db3..635288fc36 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -10,12 +10,12 @@ #include <limits> -void initialize() +void initialize_eval_script() { static const ECCVerifyHandle verify_handle; } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(eval_script, initialize_eval_script) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); diff --git a/src/test/fuzz/fee_rate.cpp b/src/test/fuzz/fee_rate.cpp index f3d44d9f93..2955213635 100644 --- a/src/test/fuzz/fee_rate.cpp +++ b/src/test/fuzz/fee_rate.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(fee_rate) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CAmount satoshis_per_k = ConsumeMoney(fuzzed_data_provider); diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index ce8700befa..61c7681bf9 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(fees) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; diff --git a/src/test/fuzz/flatfile.cpp b/src/test/fuzz/flatfile.cpp index 95dabb8bab..d142e374b1 100644 --- a/src/test/fuzz/flatfile.cpp +++ b/src/test/fuzz/flatfile.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(flatfile) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); std::optional<FlatFilePos> flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); diff --git a/src/test/fuzz/float.cpp b/src/test/fuzz/float.cpp index a24bae5b35..d18a87d177 100644 --- a/src/test/fuzz/float.cpp +++ b/src/test/fuzz/float.cpp @@ -12,7 +12,7 @@ #include <cassert> #include <cstdint> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(float) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 753cfffdcb..fd87667755 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -5,6 +5,7 @@ #include <test/fuzz/fuzz.h> #include <test/util/setup_common.h> +#include <util/check.h> #include <cstdint> #include <unistd.h> @@ -12,6 +13,36 @@ const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; +std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>>& FuzzTargets() +{ + static std::map<std::string_view, std::tuple<TypeTestOneInput, TypeInitialize>> g_fuzz_targets; + return g_fuzz_targets; +} + +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init) +{ + const auto it_ins = FuzzTargets().try_emplace(name, std::move(target), std::move(init)); + Assert(it_ins.second); +} + +static TypeTestOneInput* g_test_one_input{nullptr}; + +void initialize() +{ + if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { + for (const auto& t : FuzzTargets()) { + std::cout << t.first << std::endl; + } + Assert(false); + } + std::string_view fuzz_target{Assert(std::getenv("FUZZ"))}; + const auto it = FuzzTargets().find(fuzz_target); + Assert(it != FuzzTargets().end()); + Assert(!g_test_one_input); + g_test_one_input = &std::get<0>(it->second); + std::get<1>(it->second)(); +} + #if defined(PROVIDE_MAIN_FUNCTION) static bool read_stdin(std::vector<uint8_t>& data) { @@ -24,14 +55,10 @@ static bool read_stdin(std::vector<uint8_t>& data) } #endif -// Default initialization: Override using a non-weak initialize(). -__attribute__((weak)) void initialize() -{ -} - // This function is used by libFuzzer extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + static const auto& test_one_input = *Assert(g_test_one_input); const std::vector<uint8_t> input(data, data + size); test_one_input(input); return 0; @@ -48,6 +75,7 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) __attribute__((weak)) int main(int argc, char** argv) { initialize(); + static const auto& test_one_input = *Assert(g_test_one_input); #ifdef __AFL_INIT // Enable AFL deferred forkserver mode. Requires compilation using // afl-clang-fast++. See fuzzing.md for details. diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 3be202b16e..52841e069a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -1,14 +1,33 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_FUZZ_FUZZ_H #define BITCOIN_TEST_FUZZ_FUZZ_H -#include <stdint.h> +#include <cstdint> +#include <functional> +#include <string_view> #include <vector> -void initialize(); -void test_one_input(const std::vector<uint8_t>& buffer); +using TypeTestOneInput = std::function<void(const std::vector<uint8_t>&)>; +using TypeInitialize = std::function<void()>; + +void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, TypeInitialize init); + +inline void FuzzFrameworkEmptyFun() {} + +#define FUZZ_TARGET(name) \ + FUZZ_TARGET_INIT(name, FuzzFrameworkEmptyFun) + +#define FUZZ_TARGET_INIT(name, init_fun) \ + void name##_fuzz_target(const std::vector<uint8_t>&); \ + struct name##_Before_Main { \ + name##_Before_Main() \ + { \ + FuzzFrameworkRegisterTarget(#name, name##_fuzz_target, init_fun); \ + } \ + } const static g_##name##_before_main; \ + void name##_fuzz_target(const std::vector<uint8_t>& buffer) #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index a9f450b0c4..c99bf940c7 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -54,7 +54,7 @@ std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_ } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(golomb_rice) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); std::vector<uint8_t> golomb_rice_data; diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 6a8699fd0f..cc1bc1c8cf 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -16,12 +16,12 @@ #include <string> #include <vector> -void initialize() +void initialize_hex() { static const ECCVerifyHandle verify_handle; } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(hex, initialize_hex) { const std::string random_hex_string(buffer.begin(), buffer.end()); const std::vector<unsigned char> data = ParseHex(random_hex_string); diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index 36d44e361f..e3b62032bc 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -39,7 +39,7 @@ extern "C" int evhttp_parse_headers_(struct evhttp_request*, struct evbuffer*); std::string RequestMethodString(HTTPRequest::RequestMethod m); -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(http_request) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; evhttp_request* evreq = evhttp_request_new(nullptr, nullptr); diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 35d6804d4f..ac83d91ea0 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -40,12 +40,12 @@ #include <set> #include <vector> -void initialize() +void initialize_integer() { SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(integer, initialize_integer) { if (buffer.size() < sizeof(uint256) + sizeof(uint160)) { return; diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 955b954700..aa8f826e4a 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -26,14 +26,14 @@ #include <string> #include <vector> -void initialize() +void initialize_key() { static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(key, initialize_key) { const CKey key = [&] { CKey k; diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 62aefb650d..665ca01fa1 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -14,14 +14,14 @@ #include <string> #include <vector> -void initialize() +void initialize_key_io() { static const ECCVerifyHandle verify_handle; ECC_Start(); SelectParams(CBaseChainParams::MAIN); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(key_io, initialize_key_io) { const std::string random_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp index 82cbc00a3a..fa4024fc38 100644 --- a/src/test/fuzz/kitchen_sink.cpp +++ b/src/test/fuzz/kitchen_sink.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <merkleblock.h> +#include <policy/fees.h> #include <rpc/util.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -9,18 +11,49 @@ #include <util/error.h> #include <util/translation.h> +#include <array> #include <cstdint> #include <vector> +namespace { +constexpr TransactionError ALL_TRANSACTION_ERROR[] = { + TransactionError::OK, + TransactionError::MISSING_INPUTS, + TransactionError::ALREADY_IN_CHAIN, + TransactionError::P2P_DISABLED, + TransactionError::MEMPOOL_REJECTED, + TransactionError::MEMPOOL_ERROR, + TransactionError::INVALID_PSBT, + TransactionError::PSBT_MISMATCH, + TransactionError::SIGHASH_MISMATCH, + TransactionError::MAX_FEE_EXCEEDED, +}; +}; // namespace + // The fuzzing kitchen sink: Fuzzing harness for functions that need to be // fuzzed but a.) don't belong in any existing fuzzing harness file, and // b.) are not important enough to warrant their own fuzzing harness file. -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(kitchen_sink) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const TransactionError transaction_error = fuzzed_data_provider.PickValueInArray<TransactionError>({TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, TransactionError::INVALID_PSBT, TransactionError::PSBT_MISMATCH, TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED}); + const TransactionError transaction_error = fuzzed_data_provider.PickValueInArray(ALL_TRANSACTION_ERROR); (void)JSONRPCTransactionError(transaction_error); (void)RPCErrorFromTransactionError(transaction_error); (void)TransactionErrorString(transaction_error); + + (void)StringForFeeEstimateHorizon(fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS)); + + const OutputType output_type = fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES); + const std::string& output_type_string = FormatOutputType(output_type); + OutputType output_type_parsed; + const bool parsed = ParseOutputType(output_type_string, output_type_parsed); + assert(parsed); + assert(output_type == output_type_parsed); + (void)ParseOutputType(fuzzed_data_provider.ConsumeRandomLengthString(64), output_type_parsed); + + const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + const std::vector<bool> bits = BytesToBits(bytes); + const std::vector<uint8_t> bytes_decoded = BitsToBytes(bits); + assert(bytes == bytes_decoded); } diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index d9de9d9866..c428a86631 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -13,12 +13,12 @@ #include <cstdint> #include <vector> -void initialize() +void initialize_load_external_block_file() { InitializeFuzzingContext(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 2b181c6da1..5b1acae57b 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -35,7 +35,7 @@ bool IsAvailableLocale(const std::string& locale_identifier) } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(locale) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string locale_identifier = ConsumeLocaleIdentifier(fuzzed_data_provider); diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 4710e75757..15bcfab3ad 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(merkleblock) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CPartialMerkleTree partial_merkle_tree; diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index fa0322a391..06cd0afe2a 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -16,14 +16,14 @@ #include <string> #include <vector> -void initialize() +void initialize_message() { static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(message, initialize_message) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string random_message = fuzzed_data_provider.ConsumeRandomLengthString(1024); diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp new file mode 100644 index 0000000000..8f843ca773 --- /dev/null +++ b/src/test/fuzz/muhash.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <crypto/muhash.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <vector> + +FUZZ_TARGET(muhash) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + std::vector<uint8_t> data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (data.empty()) { + data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + if (data2.empty()) { + data2.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + } + + data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + MuHash3072 muhash; + + // Test that MuHash result is consistent independent of order of operations + muhash.Insert(data); + muhash.Insert(data2); + + uint256 out; + muhash.Finalize(out); + + muhash = MuHash3072(); + muhash.Insert(data2); + muhash.Insert(data); + + uint256 out2; + muhash.Finalize(out2); + + assert(out == out2); + + // Test that removing all added elements brings the object back to it's initial state + muhash /= muhash; + muhash.Finalize(out); + + MuHash3072 muhash2; + muhash2.Finalize(out2); + + assert(out == out2); +} diff --git a/src/test/fuzz/multiplication_overflow.cpp b/src/test/fuzz/multiplication_overflow.cpp index a4b158c18b..0f054529a6 100644 --- a/src/test/fuzz/multiplication_overflow.cpp +++ b/src/test/fuzz/multiplication_overflow.cpp @@ -14,7 +14,7 @@ #if __has_builtin(__builtin_mul_overflow) #define HAVE_BUILTIN_MUL_OVERFLOW #endif -#elif defined(__GNUC__) && (__GNUC__ >= 5) +#elif defined(__GNUC__) #define HAVE_BUILTIN_MUL_OVERFLOW #endif @@ -40,7 +40,7 @@ void TestMultiplicationOverflow(FuzzedDataProvider& fuzzed_data_provider) } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(multiplication_overflow) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); TestMultiplicationOverflow<int64_t>(fuzzed_data_provider); diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index c61d406291..3ca921b5cf 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -13,43 +13,26 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/net.h> #include <test/util/setup_common.h> #include <cstdint> #include <string> #include <vector> -void initialize() +void initialize_net() { static const BasicTestingSetup basic_testing_setup; } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(net, initialize_net) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - - const std::optional<CAddress> address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - if (!address) { - return; - } - const std::optional<CAddress> address_bind = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - if (!address_bind) { - return; - } - - CNode node{fuzzed_data_provider.ConsumeIntegral<NodeId>(), - static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()), - fuzzed_data_provider.ConsumeIntegral<int>(), - INVALID_SOCKET, - *address, - fuzzed_data_provider.ConsumeIntegral<uint64_t>(), - fuzzed_data_provider.ConsumeIntegral<uint64_t>(), - *address_bind, - fuzzed_data_provider.ConsumeRandomLengthString(32), - fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND_FULL_RELAY, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH}), - fuzzed_data_provider.ConsumeBool()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); + CNode node{ConsumeNode(fuzzed_data_provider)}; + node.SetCommonVersion(fuzzed_data_provider.ConsumeIntegral<int>()); while (fuzzed_data_provider.ConsumeBool()) { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 11)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 10)) { case 0: { node.CloseSocketDisconnect(); break; @@ -59,10 +42,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 2: { - node.SetCommonVersion(fuzzed_data_provider.ConsumeIntegral<int>()); - break; - } - case 3: { const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); if (!SanityCheckASMap(asmap)) { break; @@ -71,18 +50,18 @@ void test_one_input(const std::vector<uint8_t>& buffer) node.copyStats(stats, asmap); break; } - case 4: { + case 3: { const CNode* add_ref_node = node.AddRef(); assert(add_ref_node == &node); break; } - case 5: { + case 4: { if (node.GetRefCount() > 0) { node.Release(); } break; } - case 6: { + case 5: { if (node.m_addr_known == nullptr) { break; } @@ -93,7 +72,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) node.AddAddressKnown(*addr_opt); break; } - case 7: { + case 6: { if (node.m_addr_known == nullptr) { break; } @@ -105,7 +84,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) node.PushAddress(*addr_opt, fast_random_context); break; } - case 8: { + case 7: { const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); if (!inv_opt) { break; @@ -113,11 +92,11 @@ void test_one_input(const std::vector<uint8_t>& buffer) node.AddKnownTx(inv_opt->hash); break; } - case 9: { + case 8: { node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); break; } - case 10: { + case 9: { const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); if (!service_opt) { break; @@ -125,10 +104,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) node.SetAddrLocal(*service_opt); break; } - case 11: { + case 10: { const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); bool complete; - node.ReceiveMsgBytes((const char*)b.data(), b.size(), complete); + node.ReceiveMsgBytes(b, complete); break; } } @@ -139,15 +118,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)node.GetId(); (void)node.GetLocalNonce(); (void)node.GetLocalServices(); - (void)node.GetMyStartingHeight(); const int ref_count = node.GetRefCount(); assert(ref_count >= 0); (void)node.GetCommonVersion(); (void)node.RelayAddrsWithConn(); - const NetPermissionFlags net_permission_flags = fuzzed_data_provider.ConsumeBool() ? - fuzzed_data_provider.PickValueInArray<NetPermissionFlags>({NetPermissionFlags::PF_NONE, NetPermissionFlags::PF_BLOOMFILTER, NetPermissionFlags::PF_RELAY, NetPermissionFlags::PF_FORCERELAY, NetPermissionFlags::PF_NOBAN, NetPermissionFlags::PF_MEMPOOL, NetPermissionFlags::PF_ISIMPLICIT, NetPermissionFlags::PF_ALL}) : - static_cast<NetPermissionFlags>(fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + const NetPermissionFlags net_permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); (void)node.HasPermission(net_permission_flags); (void)node.ConnectedThroughNetwork(); } diff --git a/src/test/fuzz/net_permissions.cpp b/src/test/fuzz/net_permissions.cpp index 8a674ac1e9..544a33047b 100644 --- a/src/test/fuzz/net_permissions.cpp +++ b/src/test/fuzz/net_permissions.cpp @@ -13,22 +13,11 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(net_permissions) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string s = fuzzed_data_provider.ConsumeRandomLengthString(32); - const NetPermissionFlags net_permission_flags = fuzzed_data_provider.ConsumeBool() ? fuzzed_data_provider.PickValueInArray<NetPermissionFlags>({ - NetPermissionFlags::PF_NONE, - NetPermissionFlags::PF_BLOOMFILTER, - NetPermissionFlags::PF_RELAY, - NetPermissionFlags::PF_FORCERELAY, - NetPermissionFlags::PF_NOBAN, - NetPermissionFlags::PF_MEMPOOL, - NetPermissionFlags::PF_ADDR, - NetPermissionFlags::PF_ISIMPLICIT, - NetPermissionFlags::PF_ALL, - }) : - static_cast<NetPermissionFlags>(fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + const NetPermissionFlags net_permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); NetWhitebindPermissions net_whitebind_permissions; bilingual_str error_net_whitebind_permissions; diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 8252f38726..6e9bb47ff6 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -12,7 +12,7 @@ #include <netinet/in.h> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(netaddress) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp new file mode 100644 index 0000000000..aaebe83c0a --- /dev/null +++ b/src/test/fuzz/node_eviction.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <net.h> +#include <optional.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <optional> +#include <vector> + +FUZZ_TARGET(node_eviction) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + std::vector<NodeEvictionCandidate> eviction_candidates; + while (fuzzed_data_provider.ConsumeBool()) { + eviction_candidates.push_back({ + fuzzed_data_provider.ConsumeIntegral<NodeId>(), + fuzzed_data_provider.ConsumeIntegral<int64_t>(), + fuzzed_data_provider.ConsumeIntegral<int64_t>(), + fuzzed_data_provider.ConsumeIntegral<int64_t>(), + fuzzed_data_provider.ConsumeIntegral<int64_t>(), + fuzzed_data_provider.ConsumeBool(), + fuzzed_data_provider.ConsumeBool(), + fuzzed_data_provider.ConsumeBool(), + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + fuzzed_data_provider.ConsumeBool(), + fuzzed_data_provider.ConsumeBool(), + }); + } + // Make a copy since eviction_candidates may be in some valid but otherwise + // indeterminate state after the SelectNodeToEvict(&&) call. + const std::vector<NodeEvictionCandidate> eviction_candidates_copy = eviction_candidates; + const Optional<NodeId> node_to_evict = SelectNodeToEvict(std::move(eviction_candidates)); + if (node_to_evict) { + assert(std::any_of(eviction_candidates_copy.begin(), eviction_candidates_copy.end(), [&node_to_evict](const NodeEvictionCandidate& eviction_candidate) { return *node_to_evict == eviction_candidate.id; })); + } +} diff --git a/src/test/fuzz/p2p_transport_deserializer.cpp b/src/test/fuzz/p2p_transport_deserializer.cpp index 7e216e16fe..163f1b839e 100644 --- a/src/test/fuzz/p2p_transport_deserializer.cpp +++ b/src/test/fuzz/p2p_transport_deserializer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,24 +12,21 @@ #include <limits> #include <vector> -void initialize() +void initialize_p2p_transport_deserializer() { SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(p2p_transport_deserializer, initialize_p2p_transport_deserializer) { // Construct deserializer, with a dummy NodeId V1TransportDeserializer deserializer{Params(), (NodeId)0, SER_NETWORK, INIT_PROTO_VERSION}; - const char* pch = (const char*)buffer.data(); - size_t n_bytes = buffer.size(); - while (n_bytes > 0) { - const int handled = deserializer.Read(pch, n_bytes); + Span<const uint8_t> msg_bytes{buffer}; + while (msg_bytes.size() > 0) { + const int handled = deserializer.Read(msg_bytes); if (handled < 0) { break; } - pch += handled; - n_bytes -= handled; if (deserializer.Complete()) { const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; uint32_t out_err_raw_size{0}; diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index f668ca8c48..411b70230a 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ #include <cstdint> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(parse_hd_keypath) { const std::string keypath_str(buffer.begin(), buffer.end()); std::vector<uint32_t> keypath; diff --git a/src/test/fuzz/parse_iso8601.cpp b/src/test/fuzz/parse_iso8601.cpp index c86f8a853e..dcb24ac127 100644 --- a/src/test/fuzz/parse_iso8601.cpp +++ b/src/test/fuzz/parse_iso8601.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(parse_iso8601) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/parse_numbers.cpp b/src/test/fuzz/parse_numbers.cpp index 59f89dc9fb..ddd2bcfba3 100644 --- a/src/test/fuzz/parse_numbers.cpp +++ b/src/test/fuzz/parse_numbers.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #include <string> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(parse_numbers) { const std::string random_string(buffer.begin(), buffer.end()); diff --git a/src/test/fuzz/parse_script.cpp b/src/test/fuzz/parse_script.cpp index 21ac1aecf3..060ad3db12 100644 --- a/src/test/fuzz/parse_script.cpp +++ b/src/test/fuzz/parse_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <script/script.h> #include <test/fuzz/fuzz.h> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(parse_script) { const std::string script_string(buffer.begin(), buffer.end()); try { diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index a269378607..afe382ba21 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -12,13 +12,13 @@ #include <limits> #include <string> -void initialize() +void initialize_parse_univalue() { static const ECCVerifyHandle verify_handle; SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) { const std::string random_string(buffer.begin(), buffer.end()); bool valid = true; diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 6c94a47f3c..35cfc5230b 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -14,12 +14,12 @@ #include <string> #include <vector> -void initialize() +void initialize_policy_estimator() { InitializeFuzzingContext(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CBlockPolicyEstimator block_policy_estimator; @@ -66,10 +66,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)block_policy_estimator.estimateFee(fuzzed_data_provider.ConsumeIntegral<int>()); EstimationResult result; - (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); + (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); FeeCalculation fee_calculation; (void)block_policy_estimator.estimateSmartFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeBool() ? &fee_calculation : nullptr, fuzzed_data_provider.ConsumeBool()); - (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE})); + (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS)); } { FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index 0edcf201c7..8fa52143d8 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -10,12 +10,12 @@ #include <cstdint> #include <vector> -void initialize() +void initialize_policy_estimator_io() { InitializeFuzzingContext(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(policy_estimator_io, initialize_policy_estimator_io) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index b7fc72373d..02beb6eb37 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -15,12 +15,12 @@ #include <string> #include <vector> -void initialize() +void initialize_pow() { SelectParams(CBaseChainParams::MAIN); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(pow, initialize_pow) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const Consensus::Params& consensus_params = Params().GetConsensus(); diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index 626e187cbd..51956bbe9e 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -204,7 +204,7 @@ public: } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(prevector) { FuzzedDataProvider prov(buffer.data(), buffer.size()); prevector_tester<8, int> test; diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 4a0f920f58..48815c8910 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -12,7 +12,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(primitives_transaction) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CScript script = ConsumeScript(fuzzed_data_provider); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 9390399878..58637662c5 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -13,6 +13,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/net.h> #include <test/util/setup_common.h> @@ -32,19 +33,10 @@ #include <vector> namespace { - -#ifdef MESSAGE_TYPE -#define TO_STRING_(s) #s -#define TO_STRING(s) TO_STRING_(s) -const std::string LIMIT_TO_MESSAGE_TYPE{TO_STRING(MESSAGE_TYPE)}; -#else -const std::string LIMIT_TO_MESSAGE_TYPE; -#endif - const TestingSetup* g_setup; } // namespace -void initialize() +void initialize_process_message() { static TestingSetup setup{ CBaseChainParams::REGTEST, @@ -60,7 +52,7 @@ void initialize() SyncWithValidationInterfaceQueue(); } -void test_one_input(const std::vector<uint8_t>& buffer) +void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); @@ -72,19 +64,50 @@ void test_one_input(const std::vector<uint8_t>& buffer) } const bool jump_out_of_ibd{fuzzed_data_provider.ConsumeBool()}; if (jump_out_of_ibd) chainstate.JumpOutOfIbd(); - CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; - CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY).release(); + CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); + FillNode(fuzzed_data_provider, p2p_node); p2p_node.fSuccessfullyConnected = true; - p2p_node.nVersion = PROTOCOL_VERSION; - p2p_node.SetCommonVersion(PROTOCOL_VERSION); connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); + + // fuzzed_data_provider is fully consumed after this call, don't use it + CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; try { g_setup->m_node.peerman->ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTime<std::chrono::microseconds>(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } + { + LOCK(p2p_node.cs_sendProcessing); + g_setup->m_node.peerman->SendMessages(&p2p_node); + } SyncWithValidationInterfaceQueue(); LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement g_setup->m_node.connman->StopNodes(); } + +FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } +FUZZ_TARGET_INIT(process_message_addr, initialize_process_message) { fuzz_target(buffer, "addr"); } +FUZZ_TARGET_INIT(process_message_block, initialize_process_message) { fuzz_target(buffer, "block"); } +FUZZ_TARGET_INIT(process_message_blocktxn, initialize_process_message) { fuzz_target(buffer, "blocktxn"); } +FUZZ_TARGET_INIT(process_message_cmpctblock, initialize_process_message) { fuzz_target(buffer, "cmpctblock"); } +FUZZ_TARGET_INIT(process_message_feefilter, initialize_process_message) { fuzz_target(buffer, "feefilter"); } +FUZZ_TARGET_INIT(process_message_filteradd, initialize_process_message) { fuzz_target(buffer, "filteradd"); } +FUZZ_TARGET_INIT(process_message_filterclear, initialize_process_message) { fuzz_target(buffer, "filterclear"); } +FUZZ_TARGET_INIT(process_message_filterload, initialize_process_message) { fuzz_target(buffer, "filterload"); } +FUZZ_TARGET_INIT(process_message_getaddr, initialize_process_message) { fuzz_target(buffer, "getaddr"); } +FUZZ_TARGET_INIT(process_message_getblocks, initialize_process_message) { fuzz_target(buffer, "getblocks"); } +FUZZ_TARGET_INIT(process_message_getblocktxn, initialize_process_message) { fuzz_target(buffer, "getblocktxn"); } +FUZZ_TARGET_INIT(process_message_getdata, initialize_process_message) { fuzz_target(buffer, "getdata"); } +FUZZ_TARGET_INIT(process_message_getheaders, initialize_process_message) { fuzz_target(buffer, "getheaders"); } +FUZZ_TARGET_INIT(process_message_headers, initialize_process_message) { fuzz_target(buffer, "headers"); } +FUZZ_TARGET_INIT(process_message_inv, initialize_process_message) { fuzz_target(buffer, "inv"); } +FUZZ_TARGET_INIT(process_message_mempool, initialize_process_message) { fuzz_target(buffer, "mempool"); } +FUZZ_TARGET_INIT(process_message_notfound, initialize_process_message) { fuzz_target(buffer, "notfound"); } +FUZZ_TARGET_INIT(process_message_ping, initialize_process_message) { fuzz_target(buffer, "ping"); } +FUZZ_TARGET_INIT(process_message_pong, initialize_process_message) { fuzz_target(buffer, "pong"); } +FUZZ_TARGET_INIT(process_message_sendcmpct, initialize_process_message) { fuzz_target(buffer, "sendcmpct"); } +FUZZ_TARGET_INIT(process_message_sendheaders, initialize_process_message) { fuzz_target(buffer, "sendheaders"); } +FUZZ_TARGET_INIT(process_message_tx, initialize_process_message) { fuzz_target(buffer, "tx"); } +FUZZ_TARGET_INIT(process_message_verack, initialize_process_message) { fuzz_target(buffer, "verack"); } +FUZZ_TARGET_INIT(process_message_version, initialize_process_message) { fuzz_target(buffer, "version"); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 19ea92b750..db52da2f7e 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -19,7 +19,7 @@ const TestingSetup* g_setup; -void initialize() +void initialize_process_messages() { static TestingSetup setup{ CBaseChainParams::REGTEST, @@ -35,7 +35,7 @@ void initialize() SyncWithValidationInterfaceQueue(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(process_messages, initialize_process_messages) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -47,15 +47,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); for (int i = 0; i < num_peers_to_add; ++i) { - const ServiceFlags service_flags = ServiceFlags(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND_FULL_RELAY, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH}); - peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, conn_type).release()); + peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); + FillNode(fuzzed_data_provider, p2p_node); p2p_node.fSuccessfullyConnected = true; p2p_node.fPauseSend = false; - p2p_node.nVersion = PROTOCOL_VERSION; - p2p_node.SetCommonVersion(PROTOCOL_VERSION); g_setup->m_node.peerman->InitializeNode(&p2p_node); connman.AddTestNode(p2p_node); @@ -79,6 +76,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) connman.ProcessMessagesOnce(random_node); } catch (const std::ios_base::failure&) { } + { + LOCK(random_node.cs_sendProcessing); + g_setup->m_node.peerman->SendMessages(&random_node); + } } SyncWithValidationInterfaceQueue(); LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement diff --git a/src/test/fuzz/protocol.cpp b/src/test/fuzz/protocol.cpp index 78df0f89e7..572181366b 100644 --- a/src/test/fuzz/protocol.cpp +++ b/src/test/fuzz/protocol.cpp @@ -12,7 +12,7 @@ #include <stdexcept> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(protocol) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<CInv> inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 908e2b16f2..0b4588c4ce 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -17,12 +17,12 @@ #include <string> #include <vector> -void initialize() +void initialize_psbt() { static const ECCVerifyHandle verify_handle; } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(psbt, initialize_psbt) { PartiallySignedTransaction psbt_mut; const std::string raw_psbt{buffer.begin(), buffer.end()}; diff --git a/src/test/fuzz/random.cpp b/src/test/fuzz/random.cpp index 7df6594ad6..96668734fd 100644 --- a/src/test/fuzz/random.cpp +++ b/src/test/fuzz/random.cpp @@ -12,7 +12,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(random) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 1fd88a5f7b..26c89a70c3 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -15,9 +15,10 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { return; diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 623b8cff3a..6087ee964a 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -14,7 +14,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(rolling_bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 4274fa4351..d883426c81 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -29,7 +29,7 @@ #include <string> #include <vector> -void initialize() +void initialize_script() { // Fuzzers using pubkey must hold an ECCVerifyHandle. static const ECCVerifyHandle verify_handle; @@ -37,7 +37,7 @@ void initialize() SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(script, initialize_script) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); @@ -71,7 +71,22 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)IsSolvable(signing_provider, script); TxoutType which_type; - (void)IsStandard(script, which_type); + bool is_standard_ret = IsStandard(script, which_type); + if (!is_standard_ret) { + assert(which_type == TxoutType::NONSTANDARD || + which_type == TxoutType::NULL_DATA || + which_type == TxoutType::MULTISIG); + } + if (which_type == TxoutType::NONSTANDARD) { + assert(!is_standard_ret); + } + if (which_type == TxoutType::NULL_DATA) { + assert(script.IsUnspendable()); + } + if (script.IsUnspendable()) { + assert(which_type == TxoutType::NULL_DATA || + which_type == TxoutType::NONSTANDARD); + } (void)RecursiveDynamicUsage(script); @@ -82,7 +97,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)script.IsPayToScriptHash(); (void)script.IsPayToWitnessScriptHash(); (void)script.IsPushOnly(); - (void)script.IsUnspendable(); (void)script.GetSigOpCount(/* fAccurate= */ false); (void)FormatScript(script); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index d20fa43d68..2091ad5d91 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -188,9 +188,9 @@ void Test(const std::string& str) ECCVerifyHandle handle; -} +} // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(script_assets_test_minimizer) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); diff --git a/src/test/fuzz/script_bitcoin_consensus.cpp b/src/test/fuzz/script_bitcoin_consensus.cpp index 22f4b4f44a..fcd66b234e 100644 --- a/src/test/fuzz/script_bitcoin_consensus.cpp +++ b/src/test/fuzz/script_bitcoin_consensus.cpp @@ -12,7 +12,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(script_bitcoin_consensus) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::vector<uint8_t> random_bytes_1 = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/script_descriptor_cache.cpp b/src/test/fuzz/script_descriptor_cache.cpp index 4bfe61cec7..1c62c018e7 100644 --- a/src/test/fuzz/script_descriptor_cache.cpp +++ b/src/test/fuzz/script_descriptor_cache.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(script_descriptor_cache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); DescriptorCache descriptor_cache; diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 300c78fca0..ce8915ca2c 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -13,12 +13,12 @@ /** Flags that are not forbidden by an assert */ static bool IsValidFlagCombination(unsigned flags); -void initialize() +void initialize_script_flags() { static const ECCVerifyHandle verify_handle; } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(script_flags, initialize_script_flags) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); try { diff --git a/src/test/fuzz/script_interpreter.cpp b/src/test/fuzz/script_interpreter.cpp index 26d5732f24..5d59771682 100644 --- a/src/test/fuzz/script_interpreter.cpp +++ b/src/test/fuzz/script_interpreter.cpp @@ -15,7 +15,7 @@ bool CastToBool(const std::vector<unsigned char>& vch); -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(script_interpreter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); { diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index 7d24af20ac..d232e984bc 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -11,7 +11,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(script_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScript script = ConsumeScript(fuzzed_data_provider); diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 87af71897b..f7e45d6889 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -16,7 +16,7 @@ #include <string> #include <vector> -void initialize() +void initialize_script_sigcache() { static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); @@ -24,12 +24,12 @@ void initialize() InitSignatureCache(); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(script_sigcache, initialize_script_sigcache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const CTransaction tx = mutable_transaction ? CTransaction{*mutable_transaction} : CTransaction{}; + const CTransaction tx{mutable_transaction ? *mutable_transaction : CMutableTransaction{}}; const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const CAmount amount = ConsumeMoney(fuzzed_data_provider); const bool store = fuzzed_data_provider.ConsumeBool(); diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index c626f950e7..fe850a6959 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -22,14 +22,14 @@ #include <string> #include <vector> -void initialize() +void initialize_script_sign() { static const ECCVerifyHandle ecc_verify_handle; ECC_Start(); SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(script_sign, initialize_script_sign) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::vector<uint8_t> key = ConsumeRandomLengthByteVector(fuzzed_data_provider, 128); diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 68c1ae58ca..650318f13c 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -24,7 +24,7 @@ bool IsValidSubtraction(const CScriptNum& lhs, const CScriptNum& rhs) } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(scriptnum_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScriptNum script_num = ConsumeScriptNum(fuzzed_data_provider); diff --git a/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp b/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp index d4f302a8d3..0435626356 100644 --- a/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp +++ b/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp @@ -14,7 +14,7 @@ int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char* out32, const unsigned char* seckey, size_t seckeylen); int ec_seckey_export_der(const secp256k1_context* ctx, unsigned char* seckey, size_t* seckeylen, const unsigned char* key32, bool compressed); -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(secp256k1_ec_seckey_import_export_der) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; secp256k1_context* secp256k1_context_sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); diff --git a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp index ed8c7aba89..f437d53b57 100644 --- a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp +++ b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp @@ -14,7 +14,7 @@ bool SigHasLowR(const secp256k1_ecdsa_signature* sig); int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char* input, size_t inputlen); -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(secp256k1_ecdsa_signature_parse_der_lax) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const std::vector<uint8_t> signature_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index e121c89665..3e7b72805e 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void initialize() +void initialize_signature_checker() { static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); } @@ -24,7 +24,7 @@ class FuzzedSignatureChecker : public BaseSignatureChecker FuzzedDataProvider& m_fuzzed_data_provider; public: - FuzzedSignatureChecker(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider(fuzzed_data_provider) + explicit FuzzedSignatureChecker(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_data_provider(fuzzed_data_provider) { } @@ -52,7 +52,7 @@ public: }; } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(signature_checker, initialize_signature_checker) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 786f1a83fe..541322d484 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -15,12 +15,12 @@ #include <optional> #include <vector> -void initialize() +void initialize_signet() { InitializeFuzzingContext(CBaseChainParams::SIGNET); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(signet, initialize_signet) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); diff --git a/src/test/fuzz/span.cpp b/src/test/fuzz/span.cpp index f6b6e8f6f0..8f753948df 100644 --- a/src/test/fuzz/span.cpp +++ b/src/test/fuzz/span.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(span) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/spanparsing.cpp b/src/test/fuzz/spanparsing.cpp index e5bf5dd608..b8996632bc 100644 --- a/src/test/fuzz/spanparsing.cpp +++ b/src/test/fuzz/spanparsing.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <test/fuzz/fuzz.h> #include <util/spanparsing.h> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(spanparsing) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const size_t query_size = fuzzed_data_provider.ConsumeIntegral<size_t>(); diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 271062dc95..282a2cd8ca 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -33,7 +33,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(string) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string random_string_1 = fuzzed_data_provider.ConsumeRandomLengthString(32); diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp index 29064bc45c..4af0e750ce 100644 --- a/src/test/fuzz/strprintf.cpp +++ b/src/test/fuzz/strprintf.cpp @@ -13,7 +13,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(str_printf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 01b523cee4..375a8c1ed0 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -22,7 +22,7 @@ std::string GetArgumentName(const std::string& name) } } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(system) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ArgsManager args_manager{}; diff --git a/src/test/fuzz/timedata.cpp b/src/test/fuzz/timedata.cpp index a0e579a88f..d7fa66298a 100644 --- a/src/test/fuzz/timedata.cpp +++ b/src/test/fuzz/timedata.cpp @@ -11,7 +11,7 @@ #include <string> #include <vector> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(timedata) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int max_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1000); diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 4f972dea1c..13ae450756 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -21,12 +21,12 @@ #include <cassert> -void initialize() +void initialize_transaction() { SelectParams(CBaseChainParams::REGTEST); } -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET_INIT(transaction, initialize_transaction) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); try { @@ -42,7 +42,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) return CTransaction(deserialize, ds); } catch (const std::ios_base::failure&) { valid_tx = false; - return CTransaction(); + return CTransaction{CMutableTransaction{}}; } }(); bool valid_mutable_tx = true; diff --git a/src/test/fuzz/tx_in.cpp b/src/test/fuzz/tx_in.cpp index 8e116537d1..f8247c1fa4 100644 --- a/src/test/fuzz/tx_in.cpp +++ b/src/test/fuzz/tx_in.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <cassert> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(tx_in) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); CTxIn tx_in; diff --git a/src/test/fuzz/tx_out.cpp b/src/test/fuzz/tx_out.cpp index aa1338d5ba..39a50b6c80 100644 --- a/src/test/fuzz/tx_out.cpp +++ b/src/test/fuzz/tx_out.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ #include <test/fuzz/fuzz.h> #include <version.h> -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(tx_out) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); CTxOut tx_out; diff --git a/src/test/fuzz/txrequest.cpp b/src/test/fuzz/txrequest.cpp index 9529ad3274..72438ff2d7 100644 --- a/src/test/fuzz/txrequest.cpp +++ b/src/test/fuzz/txrequest.cpp @@ -310,7 +310,7 @@ public: }; } // namespace -void test_one_input(const std::vector<uint8_t>& buffer) +FUZZ_TARGET(txrequest) { // Tester object (which encapsulates a TxRequestTracker). Tester tester; diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index e99ed8d72d..4e8e501895 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -22,6 +22,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/util/net.h> #include <test/util/setup_common.h> #include <txmempool.h> #include <uint256.h> @@ -35,23 +36,23 @@ #include <string> #include <vector> -NODISCARD inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { const std::string s = fuzzed_data_provider.ConsumeRandomLengthString(max_length); return {s.begin(), s.end()}; } -NODISCARD inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { return BytesToBits(ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length)); } -NODISCARD inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { return {ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; } -NODISCARD inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept +[[nodiscard]] inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept { const size_t n_elements = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, max_vector_size); std::vector<std::string> r; @@ -62,7 +63,7 @@ NODISCARD inline std::vector<std::string> ConsumeRandomLengthStringVector(Fuzzed } template <typename T> -NODISCARD inline std::vector<T> ConsumeRandomLengthIntegralVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16) noexcept +[[nodiscard]] inline std::vector<T> ConsumeRandomLengthIntegralVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16) noexcept { const size_t n_elements = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, max_vector_size); std::vector<T> r; @@ -73,7 +74,7 @@ NODISCARD inline std::vector<T> ConsumeRandomLengthIntegralVector(FuzzedDataProv } template <typename T> -NODISCARD inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; @@ -86,35 +87,44 @@ NODISCARD inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzz return obj; } -NODISCARD inline opcodetype ConsumeOpcodeType(FuzzedDataProvider& fuzzed_data_provider) noexcept +template <typename WeakEnumType, size_t size> +[[nodiscard]] WeakEnumType ConsumeWeakEnum(FuzzedDataProvider& fuzzed_data_provider, const WeakEnumType (&all_types)[size]) noexcept +{ + return fuzzed_data_provider.ConsumeBool() ? + fuzzed_data_provider.PickValueInArray<WeakEnumType>(all_types) : + WeakEnumType(fuzzed_data_provider.ConsumeIntegral<typename std::underlying_type<WeakEnumType>::type>()); +} + +[[nodiscard]] inline opcodetype ConsumeOpcodeType(FuzzedDataProvider& fuzzed_data_provider) noexcept { return static_cast<opcodetype>(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, MAX_OPCODE)); } -NODISCARD inline CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider) noexcept { return fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, MAX_MONEY); } -NODISCARD inline int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider) noexcept { - static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:00Z"); + // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) is a no-op. + static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z"); static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z"); return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(time_min, time_max); } -NODISCARD inline CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider) noexcept { const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); return {b.begin(), b.end()}; } -NODISCARD inline CScriptNum ConsumeScriptNum(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline CScriptNum ConsumeScriptNum(FuzzedDataProvider& fuzzed_data_provider) noexcept { return CScriptNum{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; } -NODISCARD inline uint160 ConsumeUInt160(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline uint160 ConsumeUInt160(FuzzedDataProvider& fuzzed_data_provider) noexcept { const std::vector<uint8_t> v160 = fuzzed_data_provider.ConsumeBytes<uint8_t>(160 / 8); if (v160.size() != 160 / 8) { @@ -123,7 +133,7 @@ NODISCARD inline uint160 ConsumeUInt160(FuzzedDataProvider& fuzzed_data_provider return uint160{v160}; } -NODISCARD inline uint256 ConsumeUInt256(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline uint256 ConsumeUInt256(FuzzedDataProvider& fuzzed_data_provider) noexcept { const std::vector<uint8_t> v256 = fuzzed_data_provider.ConsumeBytes<uint8_t>(256 / 8); if (v256.size() != 256 / 8) { @@ -132,12 +142,12 @@ NODISCARD inline uint256 ConsumeUInt256(FuzzedDataProvider& fuzzed_data_provider return uint256{v256}; } -NODISCARD inline arith_uint256 ConsumeArithUInt256(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline arith_uint256 ConsumeArithUInt256(FuzzedDataProvider& fuzzed_data_provider) noexcept { return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } -NODISCARD inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +[[nodiscard]] inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept { // Avoid: // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' @@ -152,7 +162,7 @@ NODISCARD inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzze return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; } -NODISCARD inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept { CTxDestination tx_destination; switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 5)) { @@ -190,7 +200,7 @@ NODISCARD inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_ } template <typename T> -NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept +[[nodiscard]] bool MultiplicationOverflow(const T i, const T j) noexcept { static_assert(std::is_integral<T>::value, "Integral required."); if (std::numeric_limits<T>::is_signed) { @@ -213,7 +223,7 @@ NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept } template <class T> -NODISCARD bool AdditionOverflow(const T i, const T j) noexcept +[[nodiscard]] bool AdditionOverflow(const T i, const T j) noexcept { static_assert(std::is_integral<T>::value, "Integral required."); if (std::numeric_limits<T>::is_signed) { @@ -223,7 +233,7 @@ NODISCARD bool AdditionOverflow(const T i, const T j) noexcept return std::numeric_limits<T>::max() - i < j; } -NODISCARD inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept +[[nodiscard]] inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept { for (const CTxIn& tx_in : tx.vin) { const Coin& coin = inputs.AccessCoin(tx_in.prevout); @@ -238,7 +248,7 @@ NODISCARD inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsVie * Returns a byte vector of specified size regardless of the number of remaining bytes available * from the fuzzer. Pads with zero value bytes if needed to achieve the specified size. */ -NODISCARD inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t length) noexcept +[[nodiscard]] inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t length) noexcept { std::vector<uint8_t> result(length); const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(length); @@ -248,7 +258,7 @@ NODISCARD inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataPro return result; } -CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept { const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); CNetAddr net_addr; @@ -270,38 +280,60 @@ CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept return net_addr; } -CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept +inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; } -CService ConsumeService(FuzzedDataProvider& fuzzed_data_provider) noexcept +inline CService ConsumeService(FuzzedDataProvider& fuzzed_data_provider) noexcept { return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint16_t>()}; } -CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept +inline CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept { - return {ConsumeService(fuzzed_data_provider), static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()), fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + return {ConsumeService(fuzzed_data_provider), ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; } -CNode ConsumeNode(FuzzedDataProvider& fuzzed_data_provider) noexcept +template <bool ReturnUniquePtr = false> +auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept { - const NodeId node_id = fuzzed_data_provider.ConsumeIntegral<NodeId>(); - const ServiceFlags local_services = static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - const int my_starting_height = fuzzed_data_provider.ConsumeIntegral<int>(); + const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegral<NodeId>()); + const ServiceFlags local_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); const SOCKET socket = INVALID_SOCKET; const CAddress address = ConsumeAddress(fuzzed_data_provider); const uint64_t keyed_net_group = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); const uint64_t local_host_nonce = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); const CAddress addr_bind = ConsumeAddress(fuzzed_data_provider); const std::string addr_name = fuzzed_data_provider.ConsumeRandomLengthString(64); - const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND_FULL_RELAY, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH}); - const bool inbound_onion = fuzzed_data_provider.ConsumeBool(); - return {node_id, local_services, my_starting_height, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion}; + const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES); + const bool inbound_onion{conn_type == ConnectionType::INBOUND ? fuzzed_data_provider.ConsumeBool() : false}; + if constexpr (ReturnUniquePtr) { + return std::make_unique<CNode>(node_id, local_services, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion); + } else { + return CNode{node_id, local_services, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion}; + } +} +inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = nullopt) { return ConsumeNode<true>(fdp, node_id_in); } + +inline void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, const std::optional<int32_t>& version_in = std::nullopt) noexcept +{ + const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); + const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); + const int32_t version = version_in.value_or(fuzzed_data_provider.ConsumeIntegral<int32_t>()); + const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + + node.nServices = remote_services; + node.m_permissionFlags = permission_flags; + node.nVersion = version; + node.SetCommonVersion(version); + if (node.m_tx_relay != nullptr) { + LOCK(node.m_tx_relay->cs_filter); + node.m_tx_relay->fRelayTxes = filter_txs; + } } -void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) +inline void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) { static const BasicTestingSetup basic_testing_setup{chain_name, {"-nodebuglogfile"}}; } @@ -419,7 +451,7 @@ public: } }; -NODISCARD inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept { return {fuzzed_data_provider}; } @@ -440,7 +472,7 @@ public: } }; -NODISCARD inline FuzzedAutoFileProvider ConsumeAutoFile(FuzzedDataProvider& fuzzed_data_provider) noexcept +[[nodiscard]] inline FuzzedAutoFileProvider ConsumeAutoFile(FuzzedDataProvider& fuzzed_data_provider) noexcept { return {fuzzed_data_provider}; } diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index b0d4de89f3..73463b071e 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -17,8 +17,8 @@ BOOST_FIXTURE_TEST_SUITE(interfaces_tests, TestChain100Setup) BOOST_AUTO_TEST_CASE(findBlock) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); uint256 hash; BOOST_CHECK(chain->findBlock(active[10]->GetBlockHash(), FoundBlock().hash(hash))); @@ -44,13 +44,25 @@ BOOST_AUTO_TEST_CASE(findBlock) BOOST_CHECK(chain->findBlock(active[60]->GetBlockHash(), FoundBlock().mtpTime(mtp_time))); BOOST_CHECK_EQUAL(mtp_time, active[60]->GetMedianTimePast()); + bool cur_active{false}, next_active{false}; + uint256 next_hash; + BOOST_CHECK_EQUAL(active.Height(), 100); + BOOST_CHECK(chain->findBlock(active[99]->GetBlockHash(), FoundBlock().inActiveChain(cur_active).nextBlock(FoundBlock().inActiveChain(next_active).hash(next_hash)))); + BOOST_CHECK(cur_active); + BOOST_CHECK(next_active); + BOOST_CHECK_EQUAL(next_hash, active[100]->GetBlockHash()); + cur_active = next_active = false; + BOOST_CHECK(chain->findBlock(active[100]->GetBlockHash(), FoundBlock().inActiveChain(cur_active).nextBlock(FoundBlock().inActiveChain(next_active)))); + BOOST_CHECK(cur_active); + BOOST_CHECK(!next_active); + BOOST_CHECK(!chain->findBlock({}, FoundBlock())); } BOOST_AUTO_TEST_CASE(findFirstBlockWithTimeAndHeight) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); uint256 hash; int height; BOOST_CHECK(chain->findFirstBlockWithTimeAndHeight(/* min_time= */ 0, /* min_height= */ 5, FoundBlock().hash(hash).height(height))); @@ -59,25 +71,10 @@ BOOST_AUTO_TEST_CASE(findFirstBlockWithTimeAndHeight) BOOST_CHECK(!chain->findFirstBlockWithTimeAndHeight(/* min_time= */ active.Tip()->GetBlockTimeMax() + 1, /* min_height= */ 0)); } -BOOST_AUTO_TEST_CASE(findNextBlock) -{ - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); - bool reorg; - uint256 hash; - BOOST_CHECK(chain->findNextBlock(active[20]->GetBlockHash(), 20, FoundBlock().hash(hash), &reorg)); - BOOST_CHECK_EQUAL(hash, active[21]->GetBlockHash()); - BOOST_CHECK_EQUAL(reorg, false); - BOOST_CHECK(!chain->findNextBlock(uint256(), 20, {}, &reorg)); - BOOST_CHECK_EQUAL(reorg, true); - BOOST_CHECK(!chain->findNextBlock(active.Tip()->GetBlockHash(), active.Height(), {}, &reorg)); - BOOST_CHECK_EQUAL(reorg, false); -} - BOOST_AUTO_TEST_CASE(findAncestorByHeight) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); uint256 hash; BOOST_CHECK(chain->findAncestorByHeight(active[20]->GetBlockHash(), 10, FoundBlock().hash(hash))); BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); @@ -86,8 +83,8 @@ BOOST_AUTO_TEST_CASE(findAncestorByHeight) BOOST_AUTO_TEST_CASE(findAncestorByHash) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); int height = -1; BOOST_CHECK(chain->findAncestorByHash(active[20]->GetBlockHash(), active[10]->GetBlockHash(), FoundBlock().height(height))); BOOST_CHECK_EQUAL(height, 10); @@ -96,8 +93,8 @@ BOOST_AUTO_TEST_CASE(findAncestorByHash) BOOST_AUTO_TEST_CASE(findCommonAncestor) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); auto* orig_tip = active.Tip(); for (int i = 0; i < 10; ++i) { BlockValidationState state; @@ -126,8 +123,8 @@ BOOST_AUTO_TEST_CASE(findCommonAncestor) BOOST_AUTO_TEST_CASE(hasBlocks) { - auto chain = interfaces::MakeChain(m_node); - auto& active = ChainActive(); + auto& chain = m_node.chain; + const CChain& active = Assert(m_node.chainman)->ActiveChain(); // Test ranges BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 3362b8d17c..cb66d5164e 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -172,20 +172,30 @@ BOOST_AUTO_TEST_CASE(key_signature_tests) } BOOST_CHECK(found); - // When entropy is not specified, we should always see low R signatures that are less than 70 bytes in 256 tries + // When entropy is not specified, we should always see low R signatures that are less than or equal to 70 bytes in 256 tries + // The low R signatures should always have the value of their "length of R" byte less than or equal to 32 // We should see at least one signature that is less than 70 bytes. - found = true; bool found_small = false; + bool found_big = false; + bool bad_sign = false; for (int i = 0; i < 256; ++i) { sig.clear(); std::string msg = "A message to be signed" + ToString(i); msg_hash = Hash(msg); - BOOST_CHECK(key.Sign(msg_hash, sig)); - found = sig[3] == 0x20; - BOOST_CHECK(sig.size() <= 70); + if (!key.Sign(msg_hash, sig)) { + bad_sign = true; + break; + } + // sig.size() > 70 implies sig[3] > 32, because S is always low. + // But check both conditions anyway, just in case this implication is broken for some reason + if (sig[3] > 32 || sig.size() > 70) { + found_big = true; + break; + } found_small |= sig.size() < 70; } - BOOST_CHECK(found); + BOOST_CHECK(!bad_sign); + BOOST_CHECK(!found_big); BOOST_CHECK(found_small); } diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 9bc7cc5dab..942d54ede8 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 3de79a9f45..e967273636 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 37eca8b7ef..a1b41e17ed 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -9,6 +9,7 @@ #include <cstdint> #include <net.h> #include <netbase.h> +#include <optional.h> #include <serialize.h> #include <span.h> #include <streams.h> @@ -21,10 +22,13 @@ #include <boost/test/unit_test.hpp> +#include <algorithm> #include <ios> #include <memory> #include <string> +using namespace std::literals; + class CAddrManSerializationMock : public CAddrMan { public: @@ -73,7 +77,7 @@ public: } }; -static CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) +static CDataStream AddrmanToStream(const CAddrManSerializationMock& _addrman) { CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); ssPeersIn << Params().MessageStart(); @@ -106,8 +110,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); - BOOST_CHECK(Lookup(std::string("250.7.3.3", 9), addr3, 9999, false)); - BOOST_CHECK(!Lookup(std::string("250.7.3.3\0example.com", 21), addr3, 9999, false)); + BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); + BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); // Add three addresses to new table. CService source; @@ -177,7 +181,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) { SOCKET hSocket = INVALID_SOCKET; NodeId id = 0; - int height = 0; in_addr ipv4Addr; ipv4Addr.s_addr = 0xa0b0c001; @@ -186,7 +189,7 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) std::string pszDest; std::unique_ptr<CNode> pnode1 = MakeUnique<CNode>( - id++, NODE_NETWORK, height, hSocket, addr, + id++, NODE_NETWORK, hSocket, addr, /* nKeyedNetGroupIn = */ 0, /* nLocalHostNonceIn = */ 0, CAddress(), pszDest, ConnectionType::OUTBOUND_FULL_RELAY); @@ -196,10 +199,11 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode1->IsFeelerConn() == false); BOOST_CHECK(pnode1->IsAddrFetchConn() == false); BOOST_CHECK(pnode1->IsInboundConn() == false); + BOOST_CHECK(pnode1->IsInboundOnion() == false); BOOST_CHECK_EQUAL(pnode1->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode2 = MakeUnique<CNode>( - id++, NODE_NETWORK, height, hSocket, addr, + id++, NODE_NETWORK, hSocket, addr, /* nKeyedNetGroupIn = */ 1, /* nLocalHostNonceIn = */ 1, CAddress(), pszDest, ConnectionType::INBOUND, @@ -210,24 +214,26 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode2->IsFeelerConn() == false); BOOST_CHECK(pnode2->IsAddrFetchConn() == false); BOOST_CHECK(pnode2->IsInboundConn() == true); + BOOST_CHECK(pnode2->IsInboundOnion() == false); BOOST_CHECK_EQUAL(pnode2->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode3 = MakeUnique<CNode>( - id++, NODE_NETWORK, height, hSocket, addr, + id++, NODE_NETWORK, hSocket, addr, /* nKeyedNetGroupIn = */ 0, /* nLocalHostNonceIn = */ 0, CAddress(), pszDest, ConnectionType::OUTBOUND_FULL_RELAY, - /* inbound_onion = */ true); + /* inbound_onion = */ false); BOOST_CHECK(pnode3->IsFullOutboundConn() == true); BOOST_CHECK(pnode3->IsManualConn() == false); BOOST_CHECK(pnode3->IsBlockOnlyConn() == false); BOOST_CHECK(pnode3->IsFeelerConn() == false); BOOST_CHECK(pnode3->IsAddrFetchConn() == false); BOOST_CHECK(pnode3->IsInboundConn() == false); + BOOST_CHECK(pnode3->IsInboundOnion() == false); BOOST_CHECK_EQUAL(pnode3->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode4 = MakeUnique<CNode>( - id++, NODE_NETWORK, height, hSocket, addr, + id++, NODE_NETWORK, hSocket, addr, /* nKeyedNetGroupIn = */ 1, /* nLocalHostNonceIn = */ 1, CAddress(), pszDest, ConnectionType::INBOUND, @@ -238,6 +244,7 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK(pnode4->IsFeelerConn() == false); BOOST_CHECK(pnode4->IsAddrFetchConn() == false); BOOST_CHECK(pnode4->IsInboundConn() == true); + BOOST_CHECK(pnode4->IsInboundOnion() == true); BOOST_CHECK_EQUAL(pnode4->ConnectedThroughNetwork(), Network::NET_ONION); } @@ -602,6 +609,16 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_CHECK_EQUAL(addr.ToString(), "fc00:1:2:3:4:5:6:7"); BOOST_REQUIRE(s.empty()); + // Invalid CJDNS, wrong prefix. + s << MakeSpan(ParseHex("06" // network type (CJDNS) + "10" // address length + "aa000001000200030004000500060007" // address + )); + s >> addr; + BOOST_CHECK(addr.IsCJDNS()); + BOOST_CHECK(!addr.IsValid()); + BOOST_REQUIRE(s.empty()); + // Invalid CJDNS, with bogus length. s << MakeSpan(ParseHex("06" // network type (CJDNS) "01" // address length @@ -662,7 +679,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) in_addr ipv4AddrPeer; ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); - std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, 0, INVALID_SOCKET, addr, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY); + std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, INVALID_SOCKET, addr, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY); pnode->fSuccessfullyConnected.store(true); // the peer claims to be reaching us via IPv6 @@ -769,4 +786,147 @@ BOOST_AUTO_TEST_CASE(PoissonNextSend) g_mock_deterministic_tests = false; } +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_candidates, FastRandomContext& random_context) +{ + std::vector<NodeEvictionCandidate> candidates; + for (int id = 0; id < n_candidates; ++id) { + candidates.push_back({ + /* id */ id, + /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), + /* nMinPingUsecTime */ static_cast<int64_t>(random_context.randrange(100)), + /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), + /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), + /* fRelevantServices */ random_context.randbool(), + /* fRelayTxes */ random_context.randbool(), + /* fBloomFilter */ random_context.randbool(), + /* nKeyedNetGroup */ random_context.randrange(100), + /* prefer_evict */ random_context.randbool(), + /* m_is_local */ random_context.randbool(), + }); + } + return candidates; +} + +// Returns true if any of the node ids in node_ids are selected for eviction. +bool IsEvicted(std::vector<NodeEvictionCandidate> candidates, const std::vector<NodeId>& node_ids, FastRandomContext& random_context) +{ + Shuffle(candidates.begin(), candidates.end(), random_context); + const Optional<NodeId> evicted_node_id = SelectNodeToEvict(std::move(candidates)); + if (!evicted_node_id) { + return false; + } + return std::find(node_ids.begin(), node_ids.end(), *evicted_node_id) != node_ids.end(); +} + +// Create number_of_nodes random nodes, apply setup function candidate_setup_fn, +// apply eviction logic and then return true if any of the node ids in node_ids +// are selected for eviction. +bool IsEvicted(const int number_of_nodes, std::function<void(NodeEvictionCandidate&)> candidate_setup_fn, const std::vector<NodeId>& node_ids, FastRandomContext& random_context) +{ + std::vector<NodeEvictionCandidate> candidates = GetRandomNodeEvictionCandidates(number_of_nodes, random_context); + for (NodeEvictionCandidate& candidate : candidates) { + candidate_setup_fn(candidate); + } + return IsEvicted(candidates, node_ids, random_context); +} + +namespace { +constexpr int NODE_EVICTION_TEST_ROUNDS{10}; +constexpr int NODE_EVICTION_TEST_UP_TO_N_NODES{200}; +} // namespace + +BOOST_AUTO_TEST_CASE(node_eviction_test) +{ + FastRandomContext random_context{true}; + + for (int i = 0; i < NODE_EVICTION_TEST_ROUNDS; ++i) { + for (int number_of_nodes = 0; number_of_nodes < NODE_EVICTION_TEST_UP_TO_N_NODES; ++number_of_nodes) { + // Four nodes with the highest keyed netgroup values should be + // protected from eviction. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nKeyedNetGroup = number_of_nodes - candidate.id; + }, + {0, 1, 2, 3}, random_context)); + + // Eight nodes with the lowest minimum ping time should be protected + // from eviction. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [](NodeEvictionCandidate& candidate) { + candidate.nMinPingUsecTime = candidate.id; + }, + {0, 1, 2, 3, 4, 5, 6, 7}, random_context)); + + // Four nodes that most recently sent us novel transactions accepted + // into our mempool should be protected from eviction. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nLastTXTime = number_of_nodes - candidate.id; + }, + {0, 1, 2, 3}, random_context)); + + // Up to eight non-tx-relay peers that most recently sent us novel + // blocks should be protected from eviction. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nLastBlockTime = number_of_nodes - candidate.id; + if (candidate.id <= 7) { + candidate.fRelayTxes = false; + candidate.fRelevantServices = true; + } + }, + {0, 1, 2, 3, 4, 5, 6, 7}, random_context)); + + // Four peers that most recently sent us novel blocks should be + // protected from eviction. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nLastBlockTime = number_of_nodes - candidate.id; + }, + {0, 1, 2, 3}, random_context)); + + // Combination of the previous two tests. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nLastBlockTime = number_of_nodes - candidate.id; + if (candidate.id <= 7) { + candidate.fRelayTxes = false; + candidate.fRelevantServices = true; + } + }, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, random_context)); + + // Combination of all tests above. + BOOST_CHECK(!IsEvicted( + number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { + candidate.nKeyedNetGroup = number_of_nodes - candidate.id; // 4 protected + candidate.nMinPingUsecTime = candidate.id; // 8 protected + candidate.nLastTXTime = number_of_nodes - candidate.id; // 4 protected + candidate.nLastBlockTime = number_of_nodes - candidate.id; // 4 protected + }, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, random_context)); + + // An eviction is expected given >= 29 random eviction candidates. The eviction logic protects at most + // four peers by net group, eight by lowest ping time, four by last time of novel tx, up to eight non-tx-relay + // peers by last novel block time, and four more peers by last novel block time. + if (number_of_nodes >= 29) { + BOOST_CHECK(SelectNodeToEvict(GetRandomNodeEvictionCandidates(number_of_nodes, random_context))); + } + + // No eviction is expected given <= 20 random eviction candidates. The eviction logic protects at least + // four peers by net group, eight by lowest ping time, four by last time of novel tx and four peers by last + // novel block time. + if (number_of_nodes <= 20) { + BOOST_CHECK(!SelectNodeToEvict(GetRandomNodeEvictionCandidates(number_of_nodes, random_context))); + } + + // Cases left to test: + // * "Protect the half of the remaining nodes which have been connected the longest. [...]" + // * "Pick out up to 1/4 peers that are localhost, sorted by longest uptime. [...]" + // * "If any remaining peers are preferred for eviction consider only them. [...]" + // * "Identify the network group with the most connections and youngest member. [...]" + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index f5d26fafef..66ad7bb5ea 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -16,6 +16,8 @@ #include <boost/test/unit_test.hpp> +using namespace std::literals; + BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) static CNetAddr ResolveIP(const std::string& ip) @@ -224,8 +226,22 @@ BOOST_AUTO_TEST_CASE(subnet_test) // IPv4 address with IPv6 netmask or the other way around. BOOST_CHECK(!CSubNet(ResolveIP("1.1.1.1"), ResolveIP("ffff::")).IsValid()); BOOST_CHECK(!CSubNet(ResolveIP("::1"), ResolveIP("255.0.0.0")).IsValid()); - // Can't subnet TOR (or any other non-IPv4 and non-IPv6 network). - BOOST_CHECK(!CSubNet(ResolveIP("5wyqrzbvrdsumnok.onion"), ResolveIP("255.0.0.0")).IsValid()); + + // Create Non-IP subnets. + + const CNetAddr tor_addr{ + ResolveIP("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion")}; + + subnet = CSubNet(tor_addr); + BOOST_CHECK(subnet.IsValid()); + BOOST_CHECK_EQUAL(subnet.ToString(), tor_addr.ToString()); + BOOST_CHECK(subnet.Match(tor_addr)); + BOOST_CHECK( + !subnet.Match(ResolveIP("kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion"))); + BOOST_CHECK(!subnet.Match(ResolveIP("1.2.3.4"))); + + BOOST_CHECK(!CSubNet(tor_addr, 200).IsValid()); + BOOST_CHECK(!CSubNet(tor_addr, ResolveIP("255.0.0.0")).IsValid()); subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); @@ -431,20 +447,19 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) { CNetAddr addr; - BOOST_CHECK(LookupHost(std::string("127.0.0.1", 9), addr, false)); - BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0", 10), addr, false)); - BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com", 21), addr, false)); - BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com\0", 22), addr, false)); + BOOST_CHECK(LookupHost("127.0.0.1"s, addr, false)); + BOOST_CHECK(!LookupHost("127.0.0.1\0"s, addr, false)); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, addr, false)); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, addr, false)); CSubNet ret; - BOOST_CHECK(LookupSubNet(std::string("1.2.3.0/24", 10), ret)); - BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); - BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); - BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); - // We only do subnetting for IPv4 and IPv6 - BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); - BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); - BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); - BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); + BOOST_CHECK(LookupSubNet("1.2.3.0/24"s, ret)); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com"s, ret)); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com\0"s, ret)); + BOOST_CHECK(LookupSubNet("5wyqrzbvrdsumnok.onion"s, ret)); + BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0"s, ret)); + BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0example.com"s, ret)); + BOOST_CHECK(!LookupSubNet("5wyqrzbvrdsumnok.onion\0example.com\0"s, ret)); } // Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 1d7f4861fb..d5a4d3fd80 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 8c2712f764..c489ac04e9 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index a42608a66d..7da364d316 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -48,12 +48,14 @@ BOOST_AUTO_TEST_CASE(reverselock_errors) WAIT_LOCK(mutex, lock); #ifdef DEBUG_LOCKORDER + bool prev = g_debug_lockorder_abort; + g_debug_lockorder_abort = false; + // Make sure trying to reverse lock a previous lock fails - try { - REVERSE_LOCK(lock2); - BOOST_CHECK(false); // REVERSE_LOCK(lock2) succeeded - } catch(...) { } + BOOST_CHECK_EXCEPTION(REVERSE_LOCK(lock2), std::logic_error, HasReason("lock2 was not most recent critical section locked")); BOOST_CHECK(lock2.owns_lock()); + + g_debug_lockorder_abort = prev; #endif // Make sure trying to reverse lock an unlocked lock fails diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 9a490aaf6b..740b2c72db 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 1d6bcadf69..366385b619 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -183,23 +183,23 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) s.clear(); s << ToByteVector(pubkey) << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<PKHash>(&address) && - *boost::get<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get_if<PKHash>(&address) && + *std::get_if<PKHash>(&address) == PKHash(pubkey)); // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<PKHash>(&address) && - *boost::get<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get_if<PKHash>(&address) && + *std::get_if<PKHash>(&address) == PKHash(pubkey)); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<ScriptHash>(&address) && - *boost::get<ScriptHash>(&address) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get_if<ScriptHash>(&address) && + *std::get_if<ScriptHash>(&address) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -217,7 +217,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(ExtractDestination(s, address)); WitnessV0KeyHash keyhash; CHash160().Write(pubkey).Finalize(keyhash); - BOOST_CHECK(boost::get<WitnessV0KeyHash>(&address) && *boost::get<WitnessV0KeyHash>(&address) == keyhash); + BOOST_CHECK(std::get_if<WitnessV0KeyHash>(&address) && *std::get_if<WitnessV0KeyHash>(&address) == keyhash); // TxoutType::WITNESS_V0_SCRIPTHASH s.clear(); @@ -225,7 +225,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CSHA256().Write(redeemScript.data(), redeemScript.size()).Finalize(scripthash.begin()); s << OP_0 << ToByteVector(scripthash); BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(boost::get<WitnessV0ScriptHash>(&address) && *boost::get<WitnessV0ScriptHash>(&address) == scripthash); + BOOST_CHECK(std::get_if<WitnessV0ScriptHash>(&address) && *std::get_if<WitnessV0ScriptHash>(&address) == scripthash); // TxoutType::WITNESS_UNKNOWN with unknown version s.clear(); @@ -235,7 +235,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) unk.length = 33; unk.version = 1; std::copy(pubkey.begin(), pubkey.end(), unk.program); - BOOST_CHECK(boost::get<WitnessUnknown>(&address) && *boost::get<WitnessUnknown>(&address) == unk); + BOOST_CHECK(std::get_if<WitnessUnknown>(&address) && *std::get_if<WitnessUnknown>(&address) == unk); } BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) @@ -259,8 +259,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); // TxoutType::PUBKEYHASH s.clear(); @@ -269,8 +269,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script @@ -280,8 +280,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(boost::get<ScriptHash>(&addresses[0]) && - *boost::get<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get_if<ScriptHash>(&addresses[0]) && + *std::get_if<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -293,10 +293,10 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); BOOST_CHECK_EQUAL(addresses.size(), 2U); BOOST_CHECK_EQUAL(nRequired, 2); - BOOST_CHECK(boost::get<PKHash>(&addresses[0]) && - *boost::get<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - BOOST_CHECK(boost::get<PKHash>(&addresses[1]) && - *boost::get<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && + *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get_if<PKHash>(&addresses[1]) && + *std::get_if<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); // TxoutType::NULL_DATA s.clear(); diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index c509a252e0..8f5ee1e8e2 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 19029ebd3c..71275f69d9 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -6,6 +6,9 @@ #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> +#include <boost/thread/mutex.hpp> + +#include <mutex> namespace { template <typename MutexType> @@ -29,6 +32,49 @@ void TestPotentialDeadLockDetected(MutexType& mutex1, MutexType& mutex2) BOOST_CHECK(!error_thrown); #endif } + +#ifdef DEBUG_LOCKORDER +template <typename MutexType> +void TestDoubleLock2(MutexType& m) +{ + ENTER_CRITICAL_SECTION(m); + LEAVE_CRITICAL_SECTION(m); +} + +template <typename MutexType> +void TestDoubleLock(bool should_throw) +{ + const bool prev = g_debug_lockorder_abort; + g_debug_lockorder_abort = false; + + MutexType m; + ENTER_CRITICAL_SECTION(m); + if (should_throw) { + BOOST_CHECK_EXCEPTION(TestDoubleLock2(m), std::logic_error, + HasReason("double lock detected")); + } else { + BOOST_CHECK_NO_THROW(TestDoubleLock2(m)); + } + LEAVE_CRITICAL_SECTION(m); + + BOOST_CHECK(LockStackEmpty()); + + g_debug_lockorder_abort = prev; +} +#endif /* DEBUG_LOCKORDER */ + +template <typename MutexType> +void TestInconsistentLockOrderDetected(MutexType& mutex1, MutexType& mutex2) NO_THREAD_SAFETY_ANALYSIS +{ + ENTER_CRITICAL_SECTION(mutex1); + ENTER_CRITICAL_SECTION(mutex2); +#ifdef DEBUG_LOCKORDER + BOOST_CHECK_EXCEPTION(LEAVE_CRITICAL_SECTION(mutex1), std::logic_error, HasReason("mutex1 was not most recent critical section locked")); +#endif // DEBUG_LOCKORDER + LEAVE_CRITICAL_SECTION(mutex2); + LEAVE_CRITICAL_SECTION(mutex1); + BOOST_CHECK(LockStackEmpty()); +} } // namespace BOOST_FIXTURE_TEST_SUITE(sync_tests, BasicTestingSetup) @@ -55,4 +101,48 @@ BOOST_AUTO_TEST_CASE(potential_deadlock_detected) #endif } +/* Double lock would produce an undefined behavior. Thus, we only do that if + * DEBUG_LOCKORDER is activated to detect it. We don't want non-DEBUG_LOCKORDER + * build to produce tests that exhibit known undefined behavior. */ +#ifdef DEBUG_LOCKORDER +BOOST_AUTO_TEST_CASE(double_lock_mutex) +{ + TestDoubleLock<Mutex>(true /* should throw */); +} + +BOOST_AUTO_TEST_CASE(double_lock_boost_mutex) +{ + TestDoubleLock<boost::mutex>(true /* should throw */); +} + +BOOST_AUTO_TEST_CASE(double_lock_recursive_mutex) +{ + TestDoubleLock<RecursiveMutex>(false /* should not throw */); +} +#endif /* DEBUG_LOCKORDER */ + +BOOST_AUTO_TEST_CASE(inconsistent_lock_order_detected) +{ +#ifdef DEBUG_LOCKORDER + bool prev = g_debug_lockorder_abort; + g_debug_lockorder_abort = false; +#endif // DEBUG_LOCKORDER + + RecursiveMutex rmutex1, rmutex2; + TestInconsistentLockOrderDetected(rmutex1, rmutex2); + // By checking lock order consistency (CheckLastCritical) before any unlocking (LeaveCritical) + // the lock tracking data must not have been broken by exception. + TestInconsistentLockOrderDetected(rmutex1, rmutex2); + + Mutex mutex1, mutex2; + TestInconsistentLockOrderDetected(mutex1, mutex2); + // By checking lock order consistency (CheckLastCritical) before any unlocking (LeaveCritical) + // the lock tracking data must not have been broken by exception. + TestInconsistentLockOrderDetected(mutex1, mutex2); + +#ifdef DEBUG_LOCKORDER + g_debug_lockorder_abort = prev; +#endif // DEBUG_LOCKORDER +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index a55145c738..ce555f7299 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 1f520074b1..5b35ed6976 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -762,7 +762,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // Only one TxoutType::NULL_DATA permitted in all cases t.vout.resize(2); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); + t.vout[0].nValue = 0; t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); + t.vout[1].nValue = 0; reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "multi-op-return"); diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp index 65a64f2384..c1dd47f06c 100644 --- a/src/test/util/logging.cpp +++ b/src/test/util/logging.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util/logging.h b/src/test/util/logging.h index 1fcf7ca305..ebe0ecf623 100644 --- a/src/test/util/logging.h +++ b/src/test/util/logging.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -32,7 +32,7 @@ class DebugLogHelper void check_found(); public: - DebugLogHelper(std::string message, MatchFn match = [](const std::string*){ return true; }); + explicit DebugLogHelper(std::string message, MatchFn match = [](const std::string*){ return true; }); ~DebugLogHelper() { check_found(); } }; diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 74536ae74c..0c6487fbfa 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 09f2f1807f..847a490e03 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -7,9 +7,9 @@ #include <chainparams.h> #include <net.h> -void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const +void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const { - assert(node.ReceiveMsgBytes(pch, nBytes, complete)); + assert(node.ReceiveMsgBytes(msg_bytes, complete)); if (complete) { size_t nSizeAdded = 0; auto it(node.vRecvMsg.begin()); @@ -29,11 +29,11 @@ void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const { - std::vector<unsigned char> ser_msg_header; + std::vector<uint8_t> ser_msg_header; node.m_serializer->prepareForTransport(ser_msg, ser_msg_header); bool complete; - NodeReceiveMsgBytes(node, (const char*)ser_msg_header.data(), ser_msg_header.size(), complete); - NodeReceiveMsgBytes(node, (const char*)ser_msg.data.data(), ser_msg.data.size(), complete); + NodeReceiveMsgBytes(node, ser_msg_header, complete); + NodeReceiveMsgBytes(node, ser_msg.data, complete); return complete; } diff --git a/src/test/util/net.h b/src/test/util/net.h index ca8cb7fad5..e25036be26 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -25,9 +25,40 @@ struct ConnmanTestMsg : public CConnman { void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } - void NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const; + void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const; bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const; }; +constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ + NODE_NONE, + NODE_NETWORK, + NODE_BLOOM, + NODE_WITNESS, + NODE_COMPACT_FILTERS, + NODE_NETWORK_LIMITED, +}; + +constexpr NetPermissionFlags ALL_NET_PERMISSION_FLAGS[]{ + NetPermissionFlags::PF_NONE, + NetPermissionFlags::PF_BLOOMFILTER, + NetPermissionFlags::PF_RELAY, + NetPermissionFlags::PF_FORCERELAY, + NetPermissionFlags::PF_NOBAN, + NetPermissionFlags::PF_MEMPOOL, + NetPermissionFlags::PF_ADDR, + NetPermissionFlags::PF_DOWNLOAD, + NetPermissionFlags::PF_ISIMPLICIT, + NetPermissionFlags::PF_ALL, +}; + +constexpr ConnectionType ALL_CONNECTION_TYPES[]{ + ConnectionType::INBOUND, + ConnectionType::OUTBOUND_FULL_RELAY, + ConnectionType::MANUAL, + ConnectionType::FEELER, + ConnectionType::BLOCK_RELAY, + ConnectionType::ADDR_FETCH, +}; + #endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 2d3137e1e2..738f414cd0 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -16,6 +16,7 @@ #include <net.h> #include <net_processing.h> #include <noui.h> +#include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/register.h> @@ -124,41 +125,21 @@ BasicTestingSetup::~BasicTestingSetup() ECC_Stop(); } -TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) +ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : BasicTestingSetup(chainName, extra_args) { - const CChainParams& chainparams = Params(); - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - RegisterAllCoreRPCCommands(tableRPC); - - m_node.scheduler = MakeUnique<CScheduler>(); - // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. + m_node.scheduler = MakeUnique<CScheduler>(); threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - m_node.mempool = MakeUnique<CTxMemPool>(&::feeEstimator); - m_node.mempool->setSanityCheck(1.0); + m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); + m_node.mempool = std::make_unique<CTxMemPool>(m_node.fee_estimator.get(), 1); m_node.chainman = &::g_chainman; - m_node.chainman->InitializeChainstate(*m_node.mempool); - ::ChainstateActive().InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); - assert(!::ChainstateActive().CanFlushToDisk()); - ::ChainstateActive().InitCoinsCache(1 << 23); - assert(::ChainstateActive().CanFlushToDisk()); - if (!LoadGenesisBlock(chainparams)) { - throw std::runtime_error("LoadGenesisBlock failed."); - } - - BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { - throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); - } // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; @@ -166,18 +147,9 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); } g_parallel_script_checks = true; - - m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peerman = MakeUnique<PeerManager>(chainparams, *m_node.connman, m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); - { - CConnman::Options options; - options.m_msgproc = m_node.peerman.get(); - m_node.connman->Init(options); - } } -TestingSetup::~TestingSetup() +ChainTestingSetup::~ChainTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); threadGroup.interrupt_all(); @@ -195,6 +167,41 @@ TestingSetup::~TestingSetup() pblocktree.reset(); } +TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) + : ChainTestingSetup(chainName, extra_args) +{ + const CChainParams& chainparams = Params(); + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + RegisterAllCoreRPCCommands(tableRPC); + + m_node.chainman->InitializeChainstate(*m_node.mempool); + ::ChainstateActive().InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + assert(!::ChainstateActive().CanFlushToDisk()); + ::ChainstateActive().InitCoinsCache(1 << 23); + assert(::ChainstateActive().CanFlushToDisk()); + if (!LoadGenesisBlock(chainparams)) { + throw std::runtime_error("LoadGenesisBlock failed."); + } + + BlockValidationState state; + if (!ActivateBestChain(state, chainparams)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); + } + + m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + m_node.peerman = PeerManager::make(chainparams, *m_node.connman, m_node.banman.get(), + *m_node.scheduler, *m_node.chainman, *m_node.mempool, + false); + { + CConnman::Options options; + options.m_msgproc = m_node.peerman.get(); + m_node.connman->Init(options); + } +} + TestChain100Setup::TestChain100Setup() { // Generate a 100-block chain: @@ -232,12 +239,12 @@ TestChain100Setup::~TestChain100Setup() gArgs.ForceSetArg("-segwitheight", "0"); } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const { return FromTx(MakeTransactionRef(tx)); } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const { return CTxMemPoolEntry(tx, nFee, nTime, nHeight, spendsCoinbase, sigOpCost, lp); diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 1812ce1666..dff8cf825e 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -83,14 +83,21 @@ private: const fs::path m_path_root; }; -/** Testing setup that configures a complete environment. - * Included are coins database, script check threads setup. +/** Testing setup that performs all steps up until right before + * ChainstateManager gets initialized. Meant for testing ChainstateManager + * initialization behaviour. */ -struct TestingSetup : public BasicTestingSetup { +struct ChainTestingSetup : public BasicTestingSetup { boost::thread_group threadGroup; + explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + ~ChainTestingSetup(); +}; + +/** Testing setup that configures a complete environment. + */ +struct TestingSetup : public ChainTestingSetup { explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); - ~TestingSetup(); }; /** Identical to TestingSetup, but chain set to regtest */ @@ -138,8 +145,8 @@ struct TestMemPoolEntryHelper nFee(0), nTime(0), nHeight(1), spendsCoinbase(false), sigOpCost(4) { } - CTxMemPoolEntry FromTx(const CMutableTransaction& tx); - CTxMemPoolEntry FromTx(const CTransactionRef& tx); + CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; + CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; // Change the default value TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 010b6adf1f..4133f2623b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -23,6 +23,7 @@ #include <array> #include <stdint.h> +#include <string.h> #include <thread> #include <univalue.h> #include <utility> @@ -35,6 +36,8 @@ #include <boost/test/unit_test.hpp> +using namespace std::literals; + /* defined in logging.cpp */ namespace BCLog { std::string LogEscapeMessage(const std::string& str); @@ -228,7 +231,7 @@ public: Optional<std::vector<std::string>> list_value; const char* error = nullptr; - Expect(util::SettingsValue s) : setting(std::move(s)) {} + explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} Expect& DefaultString() { default_string = true; return *this; } Expect& DefaultInt() { default_int = true; return *this; } Expect& DefaultBool() { default_bool = true; return *this; } @@ -1257,9 +1260,9 @@ BOOST_AUTO_TEST_CASE(util_ParseMoney) BOOST_CHECK(!ParseMoney("-1", ret)); // Parsing strings with embedded NUL characters should fail - BOOST_CHECK(!ParseMoney(std::string("\0-1", 3), ret)); - BOOST_CHECK(!ParseMoney(std::string("\01", 2), ret)); - BOOST_CHECK(!ParseMoney(std::string("1\0", 2), ret)); + BOOST_CHECK(!ParseMoney("\0-1"s, ret)); + BOOST_CHECK(!ParseMoney("\0" "1"s, ret)); + BOOST_CHECK(!ParseMoney("1\0"s, ret)); } BOOST_AUTO_TEST_CASE(util_IsHex) @@ -1420,10 +1423,18 @@ BOOST_AUTO_TEST_CASE(test_ParseInt32) BOOST_CHECK(ParseInt32("2147483647", &n) && n == 2147483647); BOOST_CHECK(ParseInt32("-2147483648", &n) && n == (-2147483647 - 1)); // (-2147483647 - 1) equals INT_MIN BOOST_CHECK(ParseInt32("-1234", &n) && n == -1234); + BOOST_CHECK(ParseInt32("00000000000000001234", &n) && n == 1234); + BOOST_CHECK(ParseInt32("-00000000000000001234", &n) && n == -1234); + BOOST_CHECK(ParseInt32("00000000000000000000", &n) && n == 0); + BOOST_CHECK(ParseInt32("-00000000000000000000", &n) && n == 0); // Invalid values BOOST_CHECK(!ParseInt32("", &n)); BOOST_CHECK(!ParseInt32(" 1", &n)); // no padding inside BOOST_CHECK(!ParseInt32("1 ", &n)); + BOOST_CHECK(!ParseInt32("++1", &n)); + BOOST_CHECK(!ParseInt32("+-1", &n)); + BOOST_CHECK(!ParseInt32("-+1", &n)); + BOOST_CHECK(!ParseInt32("--1", &n)); BOOST_CHECK(!ParseInt32("1a", &n)); BOOST_CHECK(!ParseInt32("aap", &n)); BOOST_CHECK(!ParseInt32("0x1", &n)); // no hex @@ -1479,10 +1490,19 @@ BOOST_AUTO_TEST_CASE(test_ParseUInt32) BOOST_CHECK(ParseUInt32("2147483647", &n) && n == 2147483647); BOOST_CHECK(ParseUInt32("2147483648", &n) && n == (uint32_t)2147483648); BOOST_CHECK(ParseUInt32("4294967295", &n) && n == (uint32_t)4294967295); + BOOST_CHECK(ParseUInt32("+1234", &n) && n == 1234); + BOOST_CHECK(ParseUInt32("00000000000000001234", &n) && n == 1234); + BOOST_CHECK(ParseUInt32("00000000000000000000", &n) && n == 0); // Invalid values + BOOST_CHECK(!ParseUInt32("-00000000000000000000", &n)); BOOST_CHECK(!ParseUInt32("", &n)); BOOST_CHECK(!ParseUInt32(" 1", &n)); // no padding inside BOOST_CHECK(!ParseUInt32(" -1", &n)); + BOOST_CHECK(!ParseUInt32("++1", &n)); + BOOST_CHECK(!ParseUInt32("+-1", &n)); + BOOST_CHECK(!ParseUInt32("-+1", &n)); + BOOST_CHECK(!ParseUInt32("--1", &n)); + BOOST_CHECK(!ParseUInt32("-1", &n)); BOOST_CHECK(!ParseUInt32("1 ", &n)); BOOST_CHECK(!ParseUInt32("1a", &n)); BOOST_CHECK(!ParseUInt32("aap", &n)); @@ -1593,9 +1613,9 @@ BOOST_AUTO_TEST_CASE(test_FormatSubVersion) std::vector<std::string> comments2; comments2.push_back(std::string("comment1")); comments2.push_back(SanitizeString(std::string("Comment2; .,_?@-; !\"#$%&'()*+/<=>[]\\^`{|}~"), SAFE_CHARS_UA_COMMENT)); // Semicolon is discouraged but not forbidden by BIP-0014 - BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, std::vector<std::string>()),std::string("/Test:0.9.99/")); - BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments),std::string("/Test:0.9.99(comment1)/")); - BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments2),std::string("/Test:0.9.99(comment1; Comment2; .,_?@-; )/")); + BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, std::vector<std::string>()),std::string("/Test:9.99.0/")); + BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments),std::string("/Test:9.99.0(comment1)/")); + BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments2),std::string("/Test:9.99.0(comment1; Comment2; .,_?@-; )/")); } BOOST_AUTO_TEST_CASE(test_ParseFixedPoint) @@ -1840,7 +1860,7 @@ BOOST_AUTO_TEST_CASE(test_Capitalize) BOOST_CHECK_EQUAL(Capitalize("\x00\xfe\xff"), "\x00\xfe\xff"); } -static std::string SpanToStr(Span<const char>& span) +static std::string SpanToStr(const Span<const char>& span) { return std::string(span.begin(), span.end()); } @@ -1994,12 +2014,6 @@ struct Tracker copies = t.copies + 1; return *this; } - Tracker& operator=(Tracker&& t) noexcept - { - origin = t.origin; - copies = t.copies; - return *this; - } }; } diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index c8a375275f..92d8cf2e7d 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 36badafc4e..3d8570e27c 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -15,21 +15,22 @@ #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) //! Basic tests for ChainstateManager. //! //! First create a legacy (IBD) chainstate, then create a snapshot chainstate. BOOST_AUTO_TEST_CASE(chainstatemanager) { - ChainstateManager manager; - CTxMemPool mempool; + ChainstateManager& manager = *m_node.chainman; + CTxMemPool& mempool = *m_node.mempool; + std::vector<CChainState*> chainstates; const CChainParams& chainparams = Params(); // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -55,7 +56,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -104,8 +105,9 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) //! Test rebalancing the caches associated with each chainstate. BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { - ChainstateManager manager; - CTxMemPool mempool; + ChainstateManager& manager = *m_node.chainman; + CTxMemPool& mempool = *m_node.mempool; + size_t max_cache = 10000; manager.m_total_coinsdb_cache = max_cache; manager.m_total_coinstip_cache = max_cache; @@ -114,7 +116,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -122,6 +124,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); + BOOST_REQUIRE(c1.LoadGenesisBlock(Params())); c1.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -131,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -139,6 +142,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); + BOOST_REQUIRE(c2.LoadGenesisBlock(Params())); c2.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index c3816af0cd..9e37f14921 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,7 +75,7 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) CMutableTransaction cb; cb.vout.emplace_back(0, CScript{}); block.vtx.push_back(MakeTransactionRef(cb)); - block.vtx.push_back(MakeTransactionRef(cb)); // Add dummy tx to excercise merkle root code + block.vtx.push_back(MakeTransactionRef(cb)); // Add dummy tx to exercise merkle root code BOOST_CHECK(!SignetTxs::Create(block, challenge)); BOOST_CHECK(!CheckSignetBlockSolution(block, signet_params->GetConsensus())); diff --git a/src/threadsafety.h b/src/threadsafety.h index 52bf83b676..28b6177927 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/tinyformat.h b/src/tinyformat.h index be63f2d5d8..bc893ccda5 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -514,7 +514,7 @@ class FormatArg { } template<typename T> - FormatArg(const T& value) + explicit FormatArg(const T& value) : m_value(static_cast<const void*>(&value)), m_formatImpl(&formatImpl<T>), m_toIntImpl(&toIntImpl<T>) @@ -970,7 +970,7 @@ class FormatListN : public FormatList public: #ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES template<typename... Args> - FormatListN(const Args&... args) + explicit FormatListN(const Args&... args) : FormatList(&m_formatterStore[0], N), m_formatterStore { FormatArg(args)... } { static_assert(sizeof...(args) == N, "Number of args must be N"); } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 8ebe3d750d..90ee9422ba 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include <chainparams.h> #include <chainparamsbase.h> +#include <compat.h> #include <crypto/hmac_sha256.h> #include <net.h> #include <netaddress.h> diff --git a/src/torcontrol.h b/src/torcontrol.h index 71a6960e54..00f19db6ae 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0c2b731967..9ae7b921b3 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -331,15 +331,10 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) - : nTransactionsUpdated(0), minerPolicyEstimator(estimator), m_epoch(0), m_has_epoch_guard(false) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator, int check_ratio) + : m_check_ratio(check_ratio), minerPolicyEstimator(estimator) { _clear(); //lock free clear - - // Sanity checks off by default for performance, because otherwise - // accepting transactions becomes O(N^2) where N is the number - // of transactions in the pool - nCheckFrequency = 0; } bool CTxMemPool::isSpent(const COutPoint& outpoint) const @@ -523,7 +518,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem if (it2 != mapTx.end()) continue; const Coin &coin = pcoins->AccessCoin(txin.prevout); - if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (m_check_ratio != 0) assert(!coin.IsSpent()); if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; @@ -619,13 +614,11 @@ static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& m void CTxMemPool::check(const CCoinsViewCache *pcoins) const { - LOCK(cs); - if (nCheckFrequency == 0) - return; + if (m_check_ratio == 0) return; - if (GetRand(std::numeric_limits<uint32_t>::max()) >= nCheckFrequency) - return; + if (GetRand(m_check_ratio) >= 1) return; + LOCK(cs); LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; @@ -1134,5 +1127,3 @@ CTxMemPool::EpochGuard::~EpochGuard() ++pool.m_epoch; pool.m_has_epoch_guard = false; } - -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index f513f14af6..9a1aa9bc2b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -15,13 +15,13 @@ #include <amount.h> #include <coins.h> -#include <crypto/siphash.h> #include <indirectmap.h> #include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <sync.h> #include <random.h> +#include <util/hasher.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/hashed_index.hpp> @@ -398,20 +398,6 @@ enum class MemPoolRemovalReason { REPLACED, //!< Removed for replacement }; -class SaltedTxidHasher -{ -private: - /** Salt */ - const uint64_t k0, k1; - -public: - SaltedTxidHasher(); - - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); - } -}; - /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -488,8 +474,8 @@ public: class CTxMemPool { private: - uint32_t nCheckFrequency GUARDED_BY(cs); //!< Value n means that n times in 2^32 we check. - std::atomic<unsigned int> nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation + const int m_check_ratio; //!< Value n means that 1 times in n we check. + std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. @@ -498,8 +484,8 @@ private: mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; mutable double rollingMinimumFeeRate; //!< minimum fee to get into the pool, decreases exponentially - mutable uint64_t m_epoch; - mutable bool m_has_epoch_guard; + mutable uint64_t m_epoch{0}; + mutable bool m_has_epoch_guard{false}; // In-memory counter for external mempool tracking purposes. // This number is incremented once every time a transaction @@ -601,8 +587,14 @@ public: std::map<uint256, CAmount> mapDeltas; /** Create a new CTxMemPool. + * Sanity checks will be off by default for performance, because otherwise + * accepting transactions becomes O(N^2) where N is the number of transactions + * in the pool. + * + * @param[in] estimator is used to estimate appropriate transaction fees. + * @param[in] check_ratio is the ratio used to determine how often sanity checks will run. */ - explicit CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); + explicit CTxMemPool(CBlockPolicyEstimator* estimator = nullptr, int check_ratio = 0); /** * If sanity-checking is turned on, check makes sure the pool is @@ -611,7 +603,6 @@ public: * check does nothing. */ void check(const CCoinsViewCache *pcoins) const; - void setSanityCheck(double dFrequency = 1.0) { LOCK(cs); nCheckFrequency = static_cast<uint32_t>(dFrequency * 4294967295.0); } // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of @@ -850,7 +841,7 @@ public: class EpochGuard { const CTxMemPool& pool; public: - EpochGuard(const CTxMemPool& in); + explicit EpochGuard(const CTxMemPool& in); ~EpochGuard(); }; // N.B. GetFreshEpoch modifies mutable state via the EpochGuard construction diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 09eb78e927..e54c073328 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -170,7 +170,7 @@ using ByTxHashView = std::tuple<const uint256&, State, Priority>; class ByTxHashViewExtractor { const PriorityComputer& m_computer; public: - ByTxHashViewExtractor(const PriorityComputer& computer) : m_computer(computer) {} + explicit ByTxHashViewExtractor(const PriorityComputer& computer) : m_computer(computer) {} using result_type = ByTxHashView; result_type operator()(const Announcement& ann) const { @@ -522,7 +522,7 @@ private: } public: - Impl(bool deterministic) : + explicit Impl(bool deterministic) : m_computer(deterministic), // Explicitly initialize m_index as we need to pass a reference to m_computer to ByTxHashViewExtractor. m_index(boost::make_tuple( diff --git a/src/uint256.cpp b/src/uint256.cpp index f358b62903..cd9cbb566a 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/uint256.h b/src/uint256.h index ceae70707e..fadf2320af 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 6080516353..048e162f7d 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -100,6 +100,10 @@ public: UniValue tmpVal(val_); return push_back(tmpVal); } + bool push_back(bool val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } bool push_back(int val_) { UniValue tmpVal(val_); return push_back(tmpVal); @@ -129,7 +133,7 @@ public: return pushKV(key, tmpVal); } bool pushKV(const std::string& key, bool val_) { - UniValue tmpVal((bool)val_); + UniValue tmpVal(val_); return pushKV(key, tmpVal); } bool pushKV(const std::string& key, int val_) { diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 70ccc0d08a..ccc1344836 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -210,19 +210,31 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK(arr.push_back((int64_t) -400LL)); BOOST_CHECK(arr.push_back((int) -401)); BOOST_CHECK(arr.push_back(-40.1)); + BOOST_CHECK(arr.push_back(true)); BOOST_CHECK_EQUAL(arr.empty(), false); - BOOST_CHECK_EQUAL(arr.size(), 9); + BOOST_CHECK_EQUAL(arr.size(), 10); BOOST_CHECK_EQUAL(arr[0].getValStr(), "1023"); + BOOST_CHECK_EQUAL(arr[0].getType(), UniValue::VNUM); BOOST_CHECK_EQUAL(arr[1].getValStr(), "zippy"); + BOOST_CHECK_EQUAL(arr[1].getType(), UniValue::VSTR); BOOST_CHECK_EQUAL(arr[2].getValStr(), "pippy"); + BOOST_CHECK_EQUAL(arr[2].getType(), UniValue::VSTR); BOOST_CHECK_EQUAL(arr[3].getValStr(), "boing"); + BOOST_CHECK_EQUAL(arr[3].getType(), UniValue::VSTR); BOOST_CHECK_EQUAL(arr[4].getValStr(), "going"); + BOOST_CHECK_EQUAL(arr[4].getType(), UniValue::VSTR); BOOST_CHECK_EQUAL(arr[5].getValStr(), "400"); + BOOST_CHECK_EQUAL(arr[5].getType(), UniValue::VNUM); BOOST_CHECK_EQUAL(arr[6].getValStr(), "-400"); + BOOST_CHECK_EQUAL(arr[6].getType(), UniValue::VNUM); BOOST_CHECK_EQUAL(arr[7].getValStr(), "-401"); + BOOST_CHECK_EQUAL(arr[7].getType(), UniValue::VNUM); BOOST_CHECK_EQUAL(arr[8].getValStr(), "-40.1"); + BOOST_CHECK_EQUAL(arr[8].getType(), UniValue::VNUM); + BOOST_CHECK_EQUAL(arr[9].getValStr(), "1"); + BOOST_CHECK_EQUAL(arr[9].getType(), UniValue::VBOOL); BOOST_CHECK_EQUAL(arr[999].getValStr(), ""); diff --git a/src/util/asmap.h b/src/util/asmap.h index b31e639bb5..d0588bc8c3 100644 --- a/src/util/asmap.h +++ b/src/util/asmap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.h b/src/util/bip32.h index 347e83db9e..8f86f2aaa6 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -10,7 +10,7 @@ #include <vector> /** Parse an HD keypaths like "m/7/0'/2000". */ -NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); +[[nodiscard]] bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); /** Write HD keypaths as strings */ std::string WriteHDKeypath(const std::vector<uint32_t>& keypath); diff --git a/src/util/check.h b/src/util/check.h index 9edf394492..bc62da3440 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -46,7 +46,7 @@ class NonFatalCheckError : public std::runtime_error #error "Cannot compile without assertions!" #endif -/** Helper for Assert(). TODO remove in C++14 and replace `decltype(get_pure_r_value(val))` with `T` (templated lambda) */ +/** Helper for Assert() */ template <typename T> T get_pure_r_value(T&& val) { @@ -54,6 +54,22 @@ T get_pure_r_value(T&& val) } /** Identity function. Abort if the value compares equal to zero */ -#define Assert(val) [&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }() +#define Assert(val) ([&]() -> decltype(get_pure_r_value(val)) { auto&& check = (val); assert(#val && check); return std::forward<decltype(get_pure_r_value(val))>(check); }()) + +/** + * Assume is the identity function. + * + * - Should be used to run non-fatal checks. In debug builds it behaves like + * Assert()/assert() to notify developers and testers about non-fatal errors. + * In production it doesn't warn or log anything. + * - For fatal errors, use Assert(). + * - For non-fatal errors in interactive sessions (e.g. RPC or command line + * interfaces), CHECK_NONFATAL() might be more appropriate. + */ +#ifdef ABORT_ON_FAILED_ASSUME +#define Assume(val) Assert(val) +#else +#define Assume(val) ((void)(val)) +#endif #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/error.cpp b/src/util/error.cpp index 6c94b80683..76fac4d391 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2019 The Bitcoin Core developers +// Copyright (c) 2010-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/error.h b/src/util/error.h index b9830c9eea..6633498d2b 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2019 The Bitcoin Core developers +// Copyright (c) 2010-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/fees.cpp b/src/util/fees.cpp index 1855c0bc90..cbefe18dbb 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/fees.h b/src/util/fees.h index 3f1c33ad9c..9ef2389d3e 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UTIL_FEES_H diff --git a/src/util/golombrice.h b/src/util/golombrice.h index 425e7f6681..67d262406f 100644 --- a/src/util/golombrice.h +++ b/src/util/golombrice.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/hasher.cpp b/src/util/hasher.cpp new file mode 100644 index 0000000000..5900daf050 --- /dev/null +++ b/src/util/hasher.cpp @@ -0,0 +1,19 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <random.h> +#include <util/hasher.h> + +#include <limits> + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +SaltedSipHasher::SaltedSipHasher() : m_k0(GetRand(std::numeric_limits<uint64_t>::max())), m_k1(GetRand(std::numeric_limits<uint64_t>::max())) {} + +size_t SaltedSipHasher::operator()(const Span<const unsigned char>& script) const +{ + return CSipHasher(m_k0, m_k1).Write(script.data(), script.size()).Finalize(); +} diff --git a/src/util/hasher.h b/src/util/hasher.h new file mode 100644 index 0000000000..fa2fea30d8 --- /dev/null +++ b/src/util/hasher.h @@ -0,0 +1,99 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_HASHER_H +#define BITCOIN_UTIL_HASHER_H + +#include <crypto/siphash.h> +#include <primitives/transaction.h> +#include <uint256.h> + +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + +class SaltedOutpointHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedOutpointHasher(); + + /** + * This *must* return size_t. With Boost 1.46 on 32-bit systems the + * unordered_map will behave unpredictably if the custom hasher returns a + * uint64_t, resulting in failures when syncing the chain (#4634). + * + * Having the hash noexcept allows libstdc++'s unordered_map to recalculate + * the hash during rehash, so it does not have to cache the value. This + * reduces node's memory by sizeof(size_t). The required recalculation has + * a slight performance penalty (around 1.6%), but this is compensated by + * memory savings of about 9% which allow for a larger dbcache setting. + * + * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html + */ + size_t operator()(const COutPoint& id) const noexcept { + return SipHashUint256Extra(k0, k1, id.hash, id.n); + } +}; + +struct FilterHeaderHasher +{ + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + +struct BlockHasher +{ + // this used to call `GetCheapHash()` in uint256, which was later moved; the + // cheap hash function simply calls ReadLE64() however, so the end result is + // identical + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + +class SaltedSipHasher +{ +private: + /** Salt */ + const uint64_t m_k0, m_k1; + +public: + SaltedSipHasher(); + + size_t operator()(const Span<const unsigned char>& script) const; +}; + +#endif // BITCOIN_UTIL_HASHER_H diff --git a/src/util/memory.h b/src/util/memory.h index 15ecf8f80d..f21b81bade 100644 --- a/src/util/memory.h +++ b/src/util/memory.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,10 +10,11 @@ #include <utility> //! Substitute for C++14 std::make_unique. +//! DEPRECATED use std::make_unique in new code. template <typename T, typename... Args> std::unique_ptr<T> MakeUnique(Args&&... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); + return std::make_unique<T>(std::forward<Args>(args)...); } #endif diff --git a/src/util/message.cpp b/src/util/message.cpp index e1d5cff48c..73948e4ff1 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -31,7 +31,7 @@ MessageVerificationResult MessageVerify( return MessageVerificationResult::ERR_INVALID_ADDRESS; } - if (boost::get<PKHash>(&destination) == nullptr) { + if (std::get_if<PKHash>(&destination) == nullptr) { return MessageVerificationResult::ERR_ADDRESS_NO_KEY; } diff --git a/src/util/moneystr.h b/src/util/moneystr.h index 9d2b6da0fc..da7f673cda 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -19,6 +19,6 @@ */ std::string FormatMoney(const CAmount& n); /** Parse an amount denoted in full coins. E.g. "0.0034" supplied on the command line. **/ -NODISCARD bool ParseMoney(const std::string& str, CAmount& nRet); +[[nodiscard]] bool ParseMoney(const std::string& str, CAmount& nRet); #endif // BITCOIN_UTIL_MONEYSTR_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 3236184b0b..f3d54a2ac9 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -280,7 +280,7 @@ std::string DecodeBase32(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -NODISCARD static bool ParsePrechecks(const std::string& str) +[[nodiscard]] static bool ParsePrechecks(const std::string& str) { if (str.empty()) // No empty string allowed return false; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 1a217dd12d..b4a61202ef 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -101,42 +101,42 @@ constexpr inline bool IsSpace(char c) noexcept { * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseInt32(const std::string& str, int32_t *out); +[[nodiscard]] bool ParseInt32(const std::string& str, int32_t *out); /** * Convert string to signed 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseInt64(const std::string& str, int64_t *out); +[[nodiscard]] bool ParseInt64(const std::string& str, int64_t *out); /** * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseUInt8(const std::string& str, uint8_t *out); +[[nodiscard]] bool ParseUInt8(const std::string& str, uint8_t *out); /** * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseUInt32(const std::string& str, uint32_t *out); +[[nodiscard]] bool ParseUInt32(const std::string& str, uint32_t *out); /** * Convert decimal string to unsigned 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseUInt64(const std::string& str, uint64_t *out); +[[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); /** * Convert string to double with strict parse error feedback. * @returns true if the entire string could be parsed as valid double, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -NODISCARD bool ParseDouble(const std::string& str, double *out); +[[nodiscard]] bool ParseDouble(const std::string& str, double *out); /** * Convert a span of bytes to a lower-case hexadecimal string. @@ -166,11 +166,11 @@ bool TimingResistantEqual(const T& a, const T& b) } /** Parse number as fixed point according to JSON number syntax. - * See http://json.org/number.gif + * See https://json.org/number.gif * @returns true on success, false on error. * @note The result must be in the range (-10^18,10^18), otherwise an overflow error will trigger. */ -NODISCARD bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out); +[[nodiscard]] bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out); /** Convert from one power-of-2 number base to another. */ template<int frombits, int tobits, bool pad, typename O, typename I> diff --git a/src/util/string.h b/src/util/string.h index a0c87bd00c..5ffdc80d88 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -15,7 +15,7 @@ #include <string> #include <vector> -NODISCARD inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") +[[nodiscard]] inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") { std::string::size_type front = str.find_first_not_of(pattern); if (front == std::string::npos) { @@ -59,7 +59,7 @@ inline std::string Join(const std::vector<std::string>& list, const std::string& /** * Check if a string does not contain any embedded NUL (\0) characters */ -NODISCARD inline bool ValidAsCString(const std::string& str) noexcept +[[nodiscard]] inline bool ValidAsCString(const std::string& str) noexcept { return str.size() == strlen(str.c_str()); } @@ -80,7 +80,7 @@ std::string ToString(const T& t) * Check whether a container begins with the given prefix. */ template <typename T1, size_t PREFIX_LEN> -NODISCARD inline bool HasPrefix(const T1& obj, +[[nodiscard]] inline bool HasPrefix(const T1& obj, const std::array<uint8_t, PREFIX_LEN>& prefix) { return obj.size() >= PREFIX_LEN && diff --git a/src/util/system.cpp b/src/util/system.cpp index 5f30136fa2..917d37ccc7 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -1047,27 +1047,36 @@ bool FileCommit(FILE *file) LogPrintf("%s: FlushFileBuffers failed: %d\n", __func__, GetLastError()); return false; } -#else - #if HAVE_FDATASYNC - if (fdatasync(fileno(file)) != 0 && errno != EINVAL) { // Ignore EINVAL for filesystems that don't support sync - LogPrintf("%s: fdatasync failed: %d\n", __func__, errno); - return false; - } - #elif defined(MAC_OSX) && defined(F_FULLFSYNC) +#elif defined(MAC_OSX) && defined(F_FULLFSYNC) if (fcntl(fileno(file), F_FULLFSYNC, 0) == -1) { // Manpage says "value other than -1" is returned on success LogPrintf("%s: fcntl F_FULLFSYNC failed: %d\n", __func__, errno); return false; } - #else +#elif HAVE_FDATASYNC + if (fdatasync(fileno(file)) != 0 && errno != EINVAL) { // Ignore EINVAL for filesystems that don't support sync + LogPrintf("%s: fdatasync failed: %d\n", __func__, errno); + return false; + } +#else if (fsync(fileno(file)) != 0 && errno != EINVAL) { LogPrintf("%s: fsync failed: %d\n", __func__, errno); return false; } - #endif #endif return true; } +void DirectoryCommit(const fs::path &dirname) +{ +#ifndef WIN32 + FILE* file = fsbridge::fopen(dirname, "r"); + if (file) { + fsync(fileno(file)); + fclose(file); + } +#endif +} + bool TruncateFile(FILE *file, unsigned int length) { #if defined(WIN32) return _chsize(_fileno(file), length) == 0; diff --git a/src/util/system.h b/src/util/system.h index 1df194ca84..010fc5b49f 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -56,11 +56,23 @@ bool error(const char* fmt, const Args&... args) } void PrintExceptionContinue(const std::exception *pex, const char* pszThread); + +/** + * Ensure file contents are fully committed to disk, using a platform-specific + * feature analogous to fsync(). + */ bool FileCommit(FILE *file); + +/** + * Sync directory contents. This is required on some environments to ensure that + * newly created files are committed to disk. + */ +void DirectoryCommit(const fs::path &dirname); + bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length); -bool RenameOver(fs::path src, fs::path dest); +[[nodiscard]] bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); @@ -188,7 +200,7 @@ protected: std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); - NODISCARD bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); + [[nodiscard]] bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); /** * Returns true if settings values from the default section should be used, @@ -220,8 +232,8 @@ public: */ void SelectConfigNetwork(const std::string& network); - NODISCARD bool ParseParameters(int argc, const char* const argv[], std::string& error); - NODISCARD bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); + [[nodiscard]] bool ParseParameters(int argc, const char* const argv[], std::string& error); + [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); /** * Log warnings for options in m_section_only_args when diff --git a/src/util/time.h b/src/util/time.h index af934e423b..c69f604dc6 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -6,9 +6,11 @@ #ifndef BITCOIN_UTIL_TIME_H #define BITCOIN_UTIL_TIME_H +#include <chrono> #include <stdint.h> #include <string> -#include <chrono> + +using namespace std::chrono_literals; void UninterruptibleSleep(const std::chrono::microseconds& n); diff --git a/src/util/translation.h b/src/util/translation.h index 695d6dac96..99899ef3c2 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/validation.cpp b/src/validation.cpp index feb7502a0f..4f7c95b2ad 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -22,7 +22,6 @@ #include <logging/timer.h> #include <node/ui_interface.h> #include <optional.h> -#include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> #include <pow.h> @@ -148,8 +147,6 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CBlockPolicyEstimator feeEstimator; - // Internal stuff namespace { CBlockIndex* pindexBestInvalid = nullptr; @@ -449,7 +446,7 @@ namespace { class MemPoolAccept { public: - MemPoolAccept(CTxMemPool& mempool) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&::ChainstateActive().CoinsTip(), m_pool), + explicit MemPoolAccept(CTxMemPool& mempool) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&::ChainstateActive().CoinsTip(), m_pool), m_limit_ancestors(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), m_limit_ancestor_size(gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), m_limit_descendants(gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), @@ -482,7 +479,7 @@ private: // All the intermediate state that gets passed between the various levels // of checking a given transaction. struct Workspace { - Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} + explicit Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} std::set<uint256> m_conflicts; CTxMemPool::setEntries m_all_conflicting; CTxMemPool::setEntries m_ancestors; @@ -505,13 +502,13 @@ private: // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size @@ -640,7 +637,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) LockPoints lp; m_view.SetBackend(m_viewmempool); - CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); + const CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); // do all inputs exist? for (const CTxIn& txin : tx.vin) { if (!coins_cache.HaveCoinInCache(txin.prevout)) { @@ -696,7 +693,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_INPUTS_NOT_STANDARD, "bad-txns-nonstandard-inputs"); } - // Check for non-standard witness in P2WSH + // Check for non-standard witnesses. if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, m_view)) return state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, "bad-witness-nonstandard"); @@ -921,7 +918,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return true; } -bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; @@ -948,7 +945,7 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute return true; } -bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, const Workspace& ws, PrecomputedTransactionData& txdata) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; @@ -1312,8 +1309,6 @@ bool CChainState::IsInitialBlockDownload() const return false; } -static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; - static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); @@ -1339,73 +1334,16 @@ static void CheckForkWarningConditions() EXCLUSIVE_LOCKS_REQUIRED(cs_main) AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial sync) - if (::ChainstateActive().IsInitialBlockDownload()) + if (::ChainstateActive().IsInitialBlockDownload()) { return; - - // If our best fork is no longer within 72 blocks (+/- 12 hours if no one mines it) - // of our head, drop it - if (pindexBestForkTip && ::ChainActive().Height() - pindexBestForkTip->nHeight >= 72) - pindexBestForkTip = nullptr; - - if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > ::ChainActive().Tip()->nChainWork + (GetBlockProof(*::ChainActive().Tip()) * 6))) - { - if (!GetfLargeWorkForkFound() && pindexBestForkBase) - { - std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + - pindexBestForkBase->phashBlock->ToString() + std::string("'"); - AlertNotify(warning); - } - if (pindexBestForkTip && pindexBestForkBase) - { - LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, - pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), - pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); - SetfLargeWorkForkFound(true); - } - else - { - LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); - SetfLargeWorkInvalidChainFound(true); - } } - else - { - SetfLargeWorkForkFound(false); - SetfLargeWorkInvalidChainFound(false); - } -} -static void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - AssertLockHeld(cs_main); - // If we are on a fork that is sufficiently large, set a warning flag - CBlockIndex* pfork = pindexNewForkTip; - CBlockIndex* plonger = ::ChainActive().Tip(); - while (pfork && pfork != plonger) - { - while (plonger && plonger->nHeight > pfork->nHeight) - plonger = plonger->pprev; - if (pfork == plonger) - break; - pfork = pfork->pprev; - } - - // We define a condition where we should warn the user about as a fork of at least 7 blocks - // with a tip within 72 blocks (+/- 12 hours if no one mines it) of ours - // We use 7 blocks rather arbitrarily as it represents just under 10% of sustained network - // hash rate operating on the fork. - // or a chain that is entirely longer than ours and invalid (note that this should be detected by both) - // We define it this way because it allows us to only store the highest fork tip (+ base) which meets - // the 7-block condition and from this always have the most-likely-to-cause-warning fork - if (pfork && (!pindexBestForkTip || pindexNewForkTip->nHeight > pindexBestForkTip->nHeight) && - pindexNewForkTip->nChainWork - pfork->nChainWork > (GetBlockProof(*pfork) * 7) && - ::ChainActive().Height() - pindexNewForkTip->nHeight < 72) - { - pindexBestForkTip = pindexNewForkTip; - pindexBestForkBase = pfork; + if (pindexBestInvalid && pindexBestInvalid->nChainWork > ::ChainActive().Tip()->nChainWork + (GetBlockProof(*::ChainActive().Tip()) * 6)) { + LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); + SetfLargeWorkInvalidChainFound(true); + } else { + SetfLargeWorkInvalidChainFound(false); } - - CheckForkWarningConditions(); } // Called both upon regular invalid block discovery *and* InvalidateBlock @@ -2457,9 +2395,7 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } bilingual_str warning_messages; - int num_unexpected_version = 0; - if (!::ChainstateActive().IsInitialBlockDownload()) - { + if (!::ChainstateActive().IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); @@ -2473,14 +2409,6 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } } } - // Check the version of the last 100 blocks to see if we need to upgrade: - for (int i = 0; i < 100 && pindex != nullptr; i++) - { - int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, chainParams.GetConsensus()); - if (pindex->nVersion > VERSIONBITS_LAST_OLD_BLOCK_VERSION && (pindex->nVersion & ~nExpectedVersion) != 0) - ++num_unexpected_version; - pindex = pindex->pprev; - } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, @@ -2488,10 +2416,6 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C FormatISO8601DateTime(pindexNew->GetBlockTime()), GuessVerificationProgress(chainParams.TxData(), pindexNew), ::ChainstateActive().CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), ::ChainstateActive().CoinsTip().GetCacheSize(), !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); - - if (num_unexpected_version > 0) { - LogPrint(BCLog::VALIDATION, "%d of last 100 blocks have unexpected version\n", num_unexpected_version); - } } /** Disconnect m_chain's tip. @@ -2746,8 +2670,8 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai AssertLockHeld(cs_main); AssertLockHeld(m_mempool.cs); - const CBlockIndex *pindexOldTip = m_chain.Tip(); - const CBlockIndex *pindexFork = m_chain.FindFork(pindexMostWork); + const CBlockIndex* pindexOldTip = m_chain.Tip(); + const CBlockIndex* pindexFork = m_chain.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; @@ -2767,7 +2691,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai fBlocksDisconnected = true; } - // Build list of new blocks to connect. + // Build list of new blocks to connect (in descending height order). std::vector<CBlockIndex*> vpindexToConnect; bool fContinue = true; int nHeight = pindexFork ? pindexFork->nHeight : -1; @@ -2777,7 +2701,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); vpindexToConnect.clear(); vpindexToConnect.reserve(nTargetHeight - nHeight); - CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); + CBlockIndex* pindexIter = pindexMostWork->GetAncestor(nTargetHeight); while (pindexIter && pindexIter->nHeight != nHeight) { vpindexToConnect.push_back(pindexIter); pindexIter = pindexIter->pprev; @@ -2785,7 +2709,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai nHeight = nTargetHeight; // Connect new blocks. - for (CBlockIndex *pindexConnect : reverse_iterate(vpindexToConnect)) { + for (CBlockIndex* pindexConnect : reverse_iterate(vpindexToConnect)) { if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. @@ -2821,11 +2745,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai } m_mempool.check(&CoinsTip()); - // Callbacks/notifications for a new best chain. - if (fInvalidFound) - CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); - else - CheckForkWarningConditions(); + CheckForkWarningConditions(); return true; } @@ -5115,15 +5035,9 @@ bool LoadMempool(CTxMemPool& pool) pool.PrioritiseTransaction(i.first, i.second); } - // TODO: remove this try except in v0.22 std::set<uint256> unbroadcast_txids; - try { - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - } catch (const std::exception&) { - // mempool.dat files created prior to v0.21 will not have an - // unbroadcast set. No need to log a failure if parsing fails here. - } + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); for (const auto& txid : unbroadcast_txids) { // Ensure transactions were accepted to mempool then add to // unbroadcast set. @@ -5187,7 +5101,9 @@ bool DumpMempool(const CTxMemPool& pool) if (!FileCommit(file.Get())) throw std::runtime_error("FileCommit failed"); file.fclose(); - RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + if (!RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat")) { + throw std::runtime_error("Rename failed"); + } int64_t last = GetTimeMicros(); LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); } catch (const std::exception& e) { diff --git a/src/validation.h b/src/validation.h index ffb038ad75..1b4e40cd53 100644 --- a/src/validation.h +++ b/src/validation.h @@ -23,6 +23,7 @@ #include <txdb.h> #include <versionbits.h> #include <serialize.h> +#include <util/hasher.h> #include <atomic> #include <map> @@ -42,7 +43,6 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; -class CBlockPolicyEstimator; class CTxMemPool; class ChainstateManager; class TxValidationState; @@ -94,14 +94,6 @@ static const unsigned int DEFAULT_CHECKLEVEL = 3; // Setting the target to >= 550 MiB will make it likely we can respect the target. static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; -struct BlockHasher -{ - // this used to call `GetCheapHash()` in uint256, which was later moved; the - // cheap hash function simply calls ReadLE64() however, so the end result is - // identical - size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } -}; - /** Current sync state passed to tip changed callbacks. */ enum class SynchronizationState { INIT_REINDEX, @@ -110,7 +102,6 @@ enum class SynchronizationState { }; extern RecursiveMutex cs_main; -extern CBlockPolicyEstimator feeEstimator; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; @@ -564,7 +555,7 @@ public: //! @returns whether or not the CoinsViews object has been fully initialized and we can //! safely flush this object to disk. - bool CanFlushToDisk() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + bool CanFlushToDisk() const EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return m_coins_views && m_coins_views->m_cacheview; } diff --git a/src/version.h b/src/version.h index 019c3a3ae7..ee646eefc3 100644 --- a/src/version.h +++ b/src/version.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbitsinfo.cpp b/src/versionbitsinfo.cpp index 20dfc044ca..fa41bad46d 100644 --- a/src/versionbitsinfo.cpp +++ b/src/versionbitsinfo.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 85aae0170d..6ed48593fb 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -53,16 +53,13 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const } /** - * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. - * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. + * @param[in] env_directory Path to environment directory * @return A shared pointer to the BerkeleyEnvironment object for the wallet directory, never empty because ~BerkeleyEnvironment * erases the weak pointer from the g_dbenvs map. * @post A new BerkeleyEnvironment weak pointer is inserted into g_dbenvs if the directory path key was not already in the map. */ -std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename) +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory) { - fs::path env_directory; - SplitWalletPath(wallet_path, env_directory, database_filename); LOCK(cs_db); auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>()); if (inserted.second) { @@ -808,21 +805,14 @@ std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(bool flush_on_close) return MakeUnique<BerkeleyBatch>(*this, false, flush_on_close); } -bool ExistsBerkeleyDatabase(const fs::path& path) -{ - fs::path env_directory; - std::string data_filename; - SplitWalletPath(path, env_directory, data_filename); - return IsBDBFile(env_directory / data_filename); -} - std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) { + fs::path data_file = BDBDataFile(path); std::unique_ptr<BerkeleyDatabase> db; { LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor - std::string data_filename; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(path, data_filename); + std::string data_filename = data_file.filename().string(); + std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path()); if (env->m_databases.count(data_filename)) { error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", (env->Directory() / data_filename).string())); status = DatabaseStatus::FAILED_ALREADY_LOADED; @@ -839,28 +829,3 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con status = DatabaseStatus::SUCCESS; return db; } - -bool IsBDBFile(const fs::path& path) -{ - if (!fs::exists(path)) return false; - - // A Berkeley DB Btree file has at least 4K. - // This check also prevents opening lock files. - boost::system::error_code ec; - auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); - if (size < 4096) return false; - - fsbridge::ifstream file(path, std::ios::binary); - if (!file.is_open()) return false; - - file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 - uint32_t data = 0; - file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic - - // Berkeley DB Btree magic bytes, from: - // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75 - // - big endian systems - 00 05 31 62 - // - little endian systems - 62 31 05 00 - return data == 0x00053162 || data == 0x62310500; -} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 9073c1b6b3..bf1617d67f 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -56,7 +56,7 @@ public: std::unordered_map<std::string, WalletDatabaseFileId> m_fileids; std::condition_variable_any m_db_in_use; - BerkeleyEnvironment(const fs::path& env_directory); + explicit BerkeleyEnvironment(const fs::path& env_directory); BerkeleyEnvironment(); ~BerkeleyEnvironment(); void Reset(); @@ -83,11 +83,8 @@ public: } }; -/** Get BerkeleyEnvironment and database filename given a wallet path. */ -std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); - -/** Check format of database file */ -bool IsBDBFile(const fs::path& path); +/** Get BerkeleyEnvironment given a directory path. */ +std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory); class BerkeleyBatch; @@ -226,9 +223,6 @@ public: std::string BerkeleyDatabaseVersion(); -//! Check if Berkeley database exists at specified path. -bool ExistsBerkeleyDatabase(const fs::path& path); - //! Return object giving access to Berkeley database at specified path. std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 1a45a2b313..a2dea84d17 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 49c1134ec6..cf9768b927 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index f2df786e2e..f7325541a9 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index bd1d114730..cd49baeb78 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -3,23 +3,130 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <chainparams.h> #include <fs.h> +#include <logging.h> #include <wallet/db.h> #include <string> -void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename) +std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) +{ + const size_t offset = wallet_dir.string().size() + 1; + std::vector<fs::path> paths; + boost::system::error_code ec; + + for (auto it = fs::recursive_directory_iterator(wallet_dir, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (ec) { + LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); + continue; + } + + try { + // Get wallet path relative to walletdir by removing walletdir from the wallet path. + // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. + const fs::path path = it->path().string().substr(offset); + + if (it->status().type() == fs::directory_file && + (IsBDBFile(BDBDataFile(it->path())) || IsSQLiteFile(SQLiteDataFile(it->path())))) { + // Found a directory which contains wallet.dat btree file, add it as a wallet. + paths.emplace_back(path); + } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && IsBDBFile(it->path())) { + if (it->path().filename() == "wallet.dat") { + // Found top-level wallet.dat btree file, add top level directory "" + // as a wallet. + paths.emplace_back(); + } else { + // Found top-level btree file not called wallet.dat. Current bitcoin + // software will never create these files but will allow them to be + // opened in a shared database environment for backwards compatibility. + // Add it to the list of available wallets. + paths.emplace_back(path); + } + } + } catch (const std::exception& e) { + LogPrintf("%s: Error scanning %s: %s\n", __func__, it->path().string(), e.what()); + it.no_push(); + } + } + + return paths; +} + +fs::path BDBDataFile(const fs::path& wallet_path) { if (fs::is_regular_file(wallet_path)) { // Special case for backwards compatibility: if wallet path points to an // existing file, treat it as the path to a BDB data file in a parent // directory that also contains BDB log files. - env_directory = wallet_path.parent_path(); - database_filename = wallet_path.filename().string(); + return wallet_path; } else { // Normal case: Interpret wallet path as a directory path containing // data and log files. - env_directory = wallet_path; - database_filename = "wallet.dat"; + return wallet_path / "wallet.dat"; } } + +fs::path SQLiteDataFile(const fs::path& path) +{ + return path / "wallet.dat"; +} + +bool IsBDBFile(const fs::path& path) +{ + if (!fs::exists(path)) return false; + + // A Berkeley DB Btree file has at least 4K. + // This check also prevents opening lock files. + boost::system::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 4096) return false; + + fsbridge::ifstream file(path, std::ios::binary); + if (!file.is_open()) return false; + + file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 + uint32_t data = 0; + file.read((char*) &data, sizeof(data)); // Read 4 bytes of file to compare against magic + + // Berkeley DB Btree magic bytes, from: + // https://github.com/file/file/blob/5824af38469ec1ca9ac3ffd251e7afe9dc11e227/magic/Magdir/database#L74-L75 + // - big endian systems - 00 05 31 62 + // - little endian systems - 62 31 05 00 + return data == 0x00053162 || data == 0x62310500; +} + +bool IsSQLiteFile(const fs::path& path) +{ + if (!fs::exists(path)) return false; + + // A SQLite Database file is at least 512 bytes. + boost::system::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (size < 512) return false; + + fsbridge::ifstream file(path, std::ios::binary); + if (!file.is_open()) return false; + + // Magic is at beginning and is 16 bytes long + char magic[16]; + file.read(magic, 16); + + // Application id is at offset 68 and 4 bytes long + file.seekg(68, std::ios::beg); + char app_id[4]; + file.read(app_id, 4); + + file.close(); + + // Check the magic, see https://sqlite.org/fileformat2.html + std::string magic_str(magic, 16); + if (magic_str != std::string("SQLite format 3", 16)) { + return false; + } + + // Check the application id matches our network magic + return memcmp(Params().MessageStart(), app_id, 4) == 0; +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 940d1cd242..2c75486a44 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -223,6 +223,14 @@ enum class DatabaseStatus { FAILED_ENCRYPT, }; +/** Recursively list database paths in directory. */ +std::vector<fs::path> ListDatabases(const fs::path& path); + std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); +fs::path BDBDataFile(const fs::path& path); +fs::path SQLiteDataFile(const fs::path& path); +bool IsBDBFile(const fs::path& path); +bool IsSQLiteFile(const fs::path& path); + #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp new file mode 100644 index 0000000000..e314107988 --- /dev/null +++ b/src/wallet/dump.cpp @@ -0,0 +1,282 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/dump.h> + +#include <util/translation.h> +#include <wallet/wallet.h> + +static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP"; +uint32_t DUMP_VERSION = 1; + +bool DumpWallet(CWallet& wallet, bilingual_str& error) +{ + // Get the dumpfile + std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + if (dump_filename.empty()) { + error = _("No dump file provided. To use dump, -dumpfile=<filename> must be provided."); + return false; + } + + fs::path path = dump_filename; + path = fs::absolute(path); + if (fs::exists(path)) { + error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), path.string()); + return false; + } + fsbridge::ofstream dump_file; + dump_file.open(path); + if (dump_file.fail()) { + error = strprintf(_("Unable to open %s for writing"), path.string()); + return false; + } + + CHashWriter hasher(0, 0); + + WalletDatabase& db = wallet.GetDatabase(); + std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(); + + bool ret = true; + if (!batch->StartCursor()) { + error = _("Error: Couldn't create cursor into database"); + ret = false; + } + + // Write out a magic string with version + std::string line = strprintf("%s,%u\n", DUMP_MAGIC, DUMP_VERSION); + dump_file.write(line.data(), line.size()); + hasher.write(line.data(), line.size()); + + // Write out the file format + line = strprintf("%s,%s\n", "format", db.Format()); + dump_file.write(line.data(), line.size()); + hasher.write(line.data(), line.size()); + + if (ret) { + + // Read the records + while (true) { + CDataStream ss_key(SER_DISK, CLIENT_VERSION); + CDataStream ss_value(SER_DISK, CLIENT_VERSION); + bool complete; + ret = batch->ReadAtCursor(ss_key, ss_value, complete); + if (complete) { + ret = true; + break; + } else if (!ret) { + error = _("Error reading next record from wallet database"); + break; + } + std::string key_str = HexStr(ss_key); + std::string value_str = HexStr(ss_value); + line = strprintf("%s,%s\n", key_str, value_str); + dump_file.write(line.data(), line.size()); + hasher.write(line.data(), line.size()); + } + } + + batch->CloseCursor(); + batch.reset(); + + // Close the wallet after we're done with it. The caller won't be doing this + wallet.Close(); + + if (ret) { + // Write the hash + tfm::format(dump_file, "checksum,%s\n", HexStr(hasher.GetHash())); + dump_file.close(); + } else { + // Remove the dumpfile on failure + dump_file.close(); + fs::remove(path); + } + + return ret; +} + +// The standard wallet deleter function blocks on the validation interface +// queue, which doesn't exist for the bitcoin-wallet. Define our own +// deleter here. +static void WalletToolReleaseWallet(CWallet* wallet) +{ + wallet->WalletLogPrintf("Releasing wallet\n"); + wallet->Close(); + delete wallet; +} + +bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +{ + // Get the dumpfile + std::string dump_filename = gArgs.GetArg("-dumpfile", ""); + if (dump_filename.empty()) { + error = _("No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided."); + return false; + } + + fs::path dump_path = dump_filename; + dump_path = fs::absolute(dump_path); + if (!fs::exists(dump_path)) { + error = strprintf(_("Dump file %s does not exist."), dump_path.string()); + return false; + } + fsbridge::ifstream dump_file(dump_path); + + // Compute the checksum + CHashWriter hasher(0, 0); + uint256 checksum; + + // Check the magic and version + std::string magic_key; + std::getline(dump_file, magic_key, ','); + std::string version_value; + std::getline(dump_file, version_value, '\n'); + if (magic_key != DUMP_MAGIC) { + error = strprintf(_("Error: Dumpfile identifier record is incorrect. Got \"%s\", expected \"%s\"."), magic_key, DUMP_MAGIC); + dump_file.close(); + return false; + } + // Check the version number (value of first record) + uint32_t ver; + if (!ParseUInt32(version_value, &ver)) { + error =strprintf(_("Error: Unable to parse version %u as a uint32_t"), version_value); + dump_file.close(); + return false; + } + if (ver != DUMP_VERSION) { + error = strprintf(_("Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version %s"), version_value); + dump_file.close(); + return false; + } + std::string magic_hasher_line = strprintf("%s,%s\n", magic_key, version_value); + hasher.write(magic_hasher_line.data(), magic_hasher_line.size()); + + // Get the stored file format + std::string format_key; + std::getline(dump_file, format_key, ','); + std::string format_value; + std::getline(dump_file, format_value, '\n'); + if (format_key != "format") { + error = strprintf(_("Error: Dumpfile format record is incorrect. Got \"%s\", expected \"format\"."), format_key); + dump_file.close(); + return false; + } + // Get the data file format with format_value as the default + std::string file_format = gArgs.GetArg("-format", format_value); + if (file_format.empty()) { + error = _("No wallet file format provided. To use createfromdump, -format=<format> must be provided."); + return false; + } + DatabaseFormat data_format; + if (file_format == "bdb") { + data_format = DatabaseFormat::BERKELEY; + } else if (file_format == "sqlite") { + data_format = DatabaseFormat::SQLITE; + } else { + error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format); + return false; + } + if (file_format != format_value) { + warnings.push_back(strprintf(_("Warning: Dumpfile wallet format \"%s\" does not match command line specified format \"%s\"."), format_value, file_format)); + } + std::string format_hasher_line = strprintf("%s,%s\n", format_key, format_value); + hasher.write(format_hasher_line.data(), format_hasher_line.size()); + + DatabaseOptions options; + DatabaseStatus status; + options.require_create = true; + options.require_format = data_format; + std::unique_ptr<WalletDatabase> database = MakeDatabase(wallet_path, options, status, error); + if (!database) return false; + + // dummy chain interface + bool ret = true; + std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet); + { + LOCK(wallet->cs_wallet); + bool first_run = true; + DBErrors load_wallet_ret = wallet->LoadWallet(first_run); + if (load_wallet_ret != DBErrors::LOAD_OK) { + error = strprintf(_("Error creating %s"), name); + return false; + } + + // Get the database handle + WalletDatabase& db = wallet->GetDatabase(); + std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(); + batch->TxnBegin(); + + // Read the records from the dump file and write them to the database + while (dump_file.good()) { + std::string key; + std::getline(dump_file, key, ','); + std::string value; + std::getline(dump_file, value, '\n'); + + if (key == "checksum") { + std::vector<unsigned char> parsed_checksum = ParseHex(value); + std::copy(parsed_checksum.begin(), parsed_checksum.end(), checksum.begin()); + break; + } + + std::string line = strprintf("%s,%s\n", key, value); + hasher.write(line.data(), line.size()); + + if (key.empty() || value.empty()) { + continue; + } + + if (!IsHex(key)) { + error = strprintf(_("Error: Got key that was not hex: %s"), key); + ret = false; + break; + } + if (!IsHex(value)) { + error = strprintf(_("Error: Got value that was not hex: %s"), value); + ret = false; + break; + } + + std::vector<unsigned char> k = ParseHex(key); + std::vector<unsigned char> v = ParseHex(value); + + CDataStream ss_key(k, SER_DISK, CLIENT_VERSION); + CDataStream ss_value(v, SER_DISK, CLIENT_VERSION); + + if (!batch->Write(ss_key, ss_value)) { + error = strprintf(_("Error: Unable to write record to new wallet")); + ret = false; + break; + } + } + + if (ret) { + uint256 comp_checksum = hasher.GetHash(); + if (checksum.IsNull()) { + error = _("Error: Missing checksum"); + ret = false; + } else if (checksum != comp_checksum) { + error = strprintf(_("Error: Dumpfile checksum does not match. Computed %s, expected %s"), HexStr(comp_checksum), HexStr(checksum)); + ret = false; + } + } + + if (ret) { + batch->TxnCommit(); + } else { + batch->TxnAbort(); + } + + batch.reset(); + + dump_file.close(); + } + wallet.reset(); // The pointer deleter will close the wallet for us. + + // Remove the wallet dir if we have a failure + if (!ret) { + fs::remove_all(wallet_path); + } + + return ret; +} diff --git a/src/wallet/dump.h b/src/wallet/dump.h new file mode 100644 index 0000000000..d0a4f5ef1d --- /dev/null +++ b/src/wallet/dump.h @@ -0,0 +1,17 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_DUMP_H +#define BITCOIN_WALLET_DUMP_H + +#include <fs.h> + +class CWallet; + +struct bilingual_str; + +bool DumpWallet(CWallet& wallet, bilingual_str& error); +bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); + +#endif // BITCOIN_WALLET_DUMP_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6cbad14de8..5e319d4f95 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -111,7 +111,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt return feebumper::Result::OK; } -static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CAmount old_fee, CCoinControl& coin_control) +static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CAmount old_fee, const CCoinControl& coin_control) { // Get the fee rate of the original transaction. This is calculated from // the tx fee/vsize, so it may have been rounded down. Add 1 satoshi to the @@ -215,7 +215,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // We cannot source new unconfirmed inputs(bip125 rule 2) new_coin_control.m_min_depth = 1; - CTransactionRef tx_new = MakeTransactionRef(); + CTransactionRef tx_new; CAmount fee_ret; int change_pos_in_out = -1; // No requested location for change bilingual_str fail_reason; @@ -231,7 +231,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // Write back transaction mtx = CMutableTransaction(*tx_new); // Mark new tx not replaceable, if requested. - if (!coin_control.m_signal_bip125_rbf.get_value_or(wallet.m_signal_rbf)) { + if (!coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf)) { for (auto& input : mtx.vin) { if (input.nSequence < 0xfffffffe) input.nSequence = 0xfffffffe; } @@ -256,7 +256,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti errors.push_back(Untranslated("Invalid or non-wallet transaction id")); return Result::MISC_ERROR; } - CWalletTx& oldWtx = it->second; + const CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime Result result = PreconditionChecks(wallet, oldWtx, errors); diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 249bc833c6..429101e774 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -49,7 +49,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr // We will use smart fee estimation unsigned int target = coin_control.m_confirm_target ? *coin_control.m_confirm_target : wallet.m_confirm_target; // By default estimates are economical iff we are signaling opt-in-RBF - bool conservative_estimate = !coin_control.m_signal_bip125_rbf.get_value_or(wallet.m_signal_rbf); + bool conservative_estimate = !coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf); // Allow to override the default fee estimate mode over the CoinControl instance if (coin_control.m_fee_mode == FeeEstimateMode::CONSERVATIVE) conservative_estimate = true; else if (coin_control.m_fee_mode == FeeEstimateMode::ECONOMICAL) conservative_estimate = false; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 8b2ef191fb..085dde1026 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -15,6 +15,9 @@ #include <util/moneystr.h> #include <util/system.h> #include <util/translation.h> +#ifdef USE_BDB +#include <wallet/bdb.h> +#endif #include <wallet/coincontrol.h> #include <wallet/wallet.h> #include <walletinitinterface.h> @@ -68,9 +71,14 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const #endif argsman.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); +#ifdef USE_BDB argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); +#else + argsman.AddHiddenArgs({"-dblogsize", "-flushwallet", "-privdb"}); +#endif + argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddHiddenArgs({"-zapwallettxes"}); diff --git a/src/interfaces/wallet.cpp b/src/wallet/interfaces.cpp index f68016b557..e4e8c50f4f 100644 --- a/src/interfaces/wallet.cpp +++ b/src/wallet/interfaces.cpp @@ -31,9 +31,22 @@ #include <utility> #include <vector> -namespace interfaces { -namespace { +using interfaces::Chain; +using interfaces::FoundBlock; +using interfaces::Handler; +using interfaces::MakeHandler; +using interfaces::Wallet; +using interfaces::WalletAddress; +using interfaces::WalletBalances; +using interfaces::WalletClient; +using interfaces::WalletOrderForm; +using interfaces::WalletTx; +using interfaces::WalletTxOut; +using interfaces::WalletTxStatus; +using interfaces::WalletValueMap; +namespace wallet { +namespace { //! Construct wallet tx struct. WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { @@ -64,7 +77,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) } //! Construct wallet tx status struct. -WalletTxStatus MakeWalletTxStatus(CWallet& wallet, const CWalletTx& wtx) +WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) { WalletTxStatus result; result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); @@ -81,7 +94,7 @@ WalletTxStatus MakeWalletTxStatus(CWallet& wallet, const CWalletTx& wtx) } //! Construct wallet TxOut struct. -WalletTxOut MakeWalletTxOut(CWallet& wallet, +WalletTxOut MakeWalletTxOut(const CWallet& wallet, const CWalletTx& wtx, int n, int depth) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) @@ -538,7 +551,7 @@ public: std::vector<std::string> listWalletDir() override { std::vector<std::string> paths; - for (auto& path : ListWalletDir()) { + for (auto& path : ListDatabases(GetWalletDir())) { paths.push_back(path.string()); } return paths; @@ -561,14 +574,14 @@ public: std::vector<std::unique_ptr<Handler>> m_rpc_handlers; std::list<CRPCCommand> m_rpc_commands; }; - } // namespace +} // namespace wallet -std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<WalletImpl>(wallet) : nullptr; } +namespace interfaces { +std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<wallet::WalletImpl>(wallet) : nullptr; } std::unique_ptr<WalletClient> MakeWalletClient(Chain& chain, ArgsManager& args) { - return MakeUnique<WalletClientImpl>(chain, args); + return MakeUnique<wallet::WalletClientImpl>(chain, args); } - } // namespace interfaces diff --git a/src/wallet/load.h b/src/wallet/load.h index e12343de27..7910f0d6e1 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6efa3d0c27..7705794c63 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -29,7 +29,6 @@ #include <util/translation.h> #include <util/url.h> #include <util/vector.h> -#include <validation.h> #include <wallet/coincontrol.h> #include <wallet/context.h> #include <wallet/feebumper.h> @@ -198,35 +197,27 @@ static std::string LabelFromValue(const UniValue& value) /** * Update coin control with fee estimation based on the given parameters * - * @param[in] pwallet Wallet pointer + * @param[in] wallet Wallet reference * @param[in,out] cc Coin control to be updated * @param[in] conf_target UniValue integer; confirmation target in blocks, values between 1 and 1008 are valid per policy/fees.h; - * if a fee_rate is present, 0 is allowed here as a no-op positional placeholder * @param[in] estimate_mode UniValue string; fee estimation mode, valid values are "unset", "economical" or "conservative"; - * if a fee_rate is present, "" is allowed here as a no-op positional placeholder * @param[in] fee_rate UniValue real; fee rate in sat/vB; - * if a fee_rate is present, both conf_target and estimate_mode must either be null, or no-op + * if present, both conf_target and estimate_mode must either be null, or "unset" * @param[in] override_min_fee bool; whether to set fOverrideFeeRate to true to disable minimum fee rate checks and instead * verify only that fee_rate is greater than 0 * @throws a JSONRPCError if conf_target, estimate_mode, or fee_rate contain invalid values or are in conflict */ -static void SetFeeEstimateMode(const CWallet* pwallet, CCoinControl& cc, const UniValue& conf_target, const UniValue& estimate_mode, const UniValue& fee_rate, bool override_min_fee) +static void SetFeeEstimateMode(const CWallet& wallet, CCoinControl& cc, const UniValue& conf_target, const UniValue& estimate_mode, const UniValue& fee_rate, bool override_min_fee) { if (!fee_rate.isNull()) { - if (!conf_target.isNull() && conf_target.get_int() > 0) { + if (!conf_target.isNull()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and fee_rate. Please provide either a confirmation target in blocks for automatic fee estimation, or an explicit fee rate."); } - if (!estimate_mode.isNull() && !estimate_mode.get_str().empty()) { + if (!estimate_mode.isNull() && estimate_mode.get_str() != "unset") { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and fee_rate"); } - CFeeRate fee_rate_in_sat_vb{CFeeRate(AmountFromValue(fee_rate), COIN)}; - if (override_min_fee) { - if (fee_rate_in_sat_vb <= CFeeRate(0)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid fee_rate %s (must be greater than 0)", fee_rate_in_sat_vb.ToString(FeeEstimateMode::SAT_VB))); - } - cc.fOverrideFeeRate = true; - } - cc.m_feerate = fee_rate_in_sat_vb; + cc.m_feerate = CFeeRate(AmountFromValue(fee_rate), COIN); + if (override_min_fee) cc.fOverrideFeeRate = true; // Default RBF to true for explicit fee_rate, if unset. if (cc.m_signal_bip125_rbf == nullopt) cc.m_signal_bip125_rbf = true; return; @@ -235,7 +226,7 @@ static void SetFeeEstimateMode(const CWallet* pwallet, CCoinControl& cc, const U throw JSONRPCError(RPC_INVALID_PARAMETER, InvalidEstimateModeErrorMessage()); } if (!conf_target.isNull()) { - cc.m_confirm_target = ParseConfirmTarget(conf_target, pwallet->chain().estimateMaxBlocks()); + cc.m_confirm_target = ParseConfirmTarget(conf_target, wallet.chain().estimateMaxBlocks()); } } @@ -318,7 +309,7 @@ static RPCHelpMan getrawchangeaddress() throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); } - OutputType output_type = pwallet->m_default_change_type.get_value_or(pwallet->m_default_address_type); + OutputType output_type = pwallet->m_default_change_type.value_or(pwallet->m_default_address_type); if (!request.params[0].isNull()) { if (!ParseOutputType(request.params[0].get_str(), output_type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); @@ -473,8 +464,8 @@ static RPCHelpMan sendtoaddress() + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1") + "\nSend 0.1 BTC with a confirmation target of 6 blocks in economical fee estimate mode using positional arguments\n" + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"donation\" \"sean's outpost\" false true 6 economical") + - "\nSend 0.1 BTC with a fee rate of 1 " + CURRENCY_ATOM + "/vB, subtract fee from amount, BIP125-replaceable, using positional arguments\n" - + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"drinks\" \"room77\" true true 0 \"\" 1") + + "\nSend 0.1 BTC with a fee rate of 1.1 " + CURRENCY_ATOM + "/vB, subtract fee from amount, BIP125-replaceable, using positional arguments\n" + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 0.1 \"drinks\" \"room77\" true true null \"unset\" null 1.1") + "\nSend 0.2 BTC with a confirmation target of 6 blocks in economical fee estimate mode using named arguments\n" + HelpExampleCli("-named sendtoaddress", "address=\"" + EXAMPLE_ADDRESS[0] + "\" amount=0.2 conf_target=6 estimate_mode=\"economical\"") + "\nSend 0.5 BTC with a fee rate of 25 " + CURRENCY_ATOM + "/vB using named arguments\n" @@ -514,7 +505,7 @@ static RPCHelpMan sendtoaddress() // We also enable partial spend avoidance if reuse avoidance is set. coin_control.m_avoid_partial_spends |= coin_control.m_avoid_address_reuse; - SetFeeEstimateMode(pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[9], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[9], /* override_min_fee */ false); EnsureWalletIsUnlocked(pwallet); @@ -636,7 +627,7 @@ static RPCHelpMan signmessage() throw JSONRPCError(RPC_TYPE_ERROR, "Invalid address"); } - const PKHash *pkhash = boost::get<PKHash>(&dest); + const PKHash* pkhash = std::get_if<PKHash>(&dest); if (!pkhash) { throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); } @@ -942,7 +933,7 @@ static RPCHelpMan sendmany() coin_control.m_signal_bip125_rbf = request.params[5].get_bool(); } - SetFeeEstimateMode(pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[8], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, /* conf_target */ request.params[6], /* estimate_mode */ request.params[7], /* fee_rate */ request.params[8], /* override_min_fee */ false); std::vector<CRecipient> recipients; ParseRecipients(sendTo, subtractFeeFromAmount, recipients); @@ -1582,8 +1573,7 @@ static RPCHelpMan listsinceblock() LOCK(wallet.cs_wallet); - // The way the 'height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> height = MakeOptional(false, int()); // Height of the specified block or the common ancestor, if the block provided was in a deactivated chain. + Optional<int> height; // Height of the specified block or the common ancestor, if the block provided was in a deactivated chain. Optional<int> altheight; // Height of the specified block, even if it's in a deactivated chain. int target_confirms = 1; isminefilter filter = ISMINE_SPENDABLE; @@ -2546,7 +2536,7 @@ static RPCHelpMan listwalletdir() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { UniValue wallets(UniValue::VARR); - for (const auto& path : ListWalletDir()) { + for (const auto& path : ListDatabases(GetWalletDir())) { UniValue wallet(UniValue::VOBJ); wallet.pushKV("name", path.string()); wallets.push_back(wallet); @@ -2764,6 +2754,12 @@ static RPCHelpMan createwallet() warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } +#ifndef USE_BDB + if (!(flags & WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without bdb support (required for legacy wallets)"); + } +#endif + DatabaseOptions options; DatabaseStatus status; options.require_create = true; @@ -2792,7 +2788,7 @@ static RPCHelpMan unloadwallet() "Unloads the wallet referenced by the request endpoint otherwise unloads the wallet specified in the argument.\n" "Specifying the wallet name on a wallet endpoint is invalid.", { - {"wallet_name", RPCArg::Type::STR, /* default */ "the wallet name from the RPC request", "The name of the wallet to unload."}, + {"wallet_name", RPCArg::Type::STR, /* default */ "the wallet name from the RPC endpoint", "The name of the wallet to unload. If provided both here and in the RPC endpoint, the two must be identical."}, {"load_on_startup", RPCArg::Type::BOOL, /* default */ "null", "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{RPCResult::Type::OBJ, "", "", { @@ -2806,8 +2802,8 @@ static RPCHelpMan unloadwallet() { std::string wallet_name; if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { - if (!request.params[0].isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot unload the requested wallet"); + if (!(request.params[0].isNull() || request.params[0].get_str() == wallet_name)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "RPC endpoint wallet and wallet_name parameter specify different wallets"); } } else { wallet_name = request.params[0].get_str(); @@ -3006,7 +3002,7 @@ static RPCHelpMan listunspent() std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); if (provider) { if (scriptPubKey.IsPayToScriptHash()) { - const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); + const CScriptID& hash = CScriptID(std::get<ScriptHash>(address)); CScript redeemScript; if (provider->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript)); @@ -3016,7 +3012,7 @@ static RPCHelpMan listunspent() bool extracted = ExtractDestination(redeemScript, witness_destination); CHECK_NONFATAL(extracted); // Also return the witness script - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); + const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(witness_destination); CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; @@ -3026,7 +3022,7 @@ static RPCHelpMan listunspent() } } } else if (scriptPubKey.IsPayToWitnessScriptHash()) { - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); + const WitnessV0ScriptHash& whash = std::get<WitnessV0ScriptHash>(address); CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; @@ -3150,11 +3146,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("estimate_mode")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and feeRate"); } - CFeeRate fee_rate(AmountFromValue(options["feeRate"])); - if (fee_rate <= CFeeRate(0)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid feeRate %s (must be greater than 0)", fee_rate.ToString(FeeEstimateMode::BTC_KVB))); - } - coinControl.m_feerate = fee_rate; + coinControl.m_feerate = CFeeRate(AmountFromValue(options["feeRate"])); coinControl.fOverrideFeeRate = true; } @@ -3164,7 +3156,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("replaceable")) { coinControl.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - SetFeeEstimateMode(pwallet, coinControl, options["conf_target"], options["estimate_mode"], options["fee_rate"], override_min_fee); + SetFeeEstimateMode(*pwallet, coinControl, options["conf_target"], options["estimate_mode"], options["fee_rate"], override_min_fee); } } else { // if options is null and not a bool @@ -3393,7 +3385,7 @@ RPCHelpMan signrawtransactionwithwallet() static RPCHelpMan bumpfee_helper(std::string method_name) { bool want_psbt = method_name == "psbtbumpfee"; - const std::string incremental_fee{CFeeRate(DEFAULT_MIN_RELAY_TX_FEE).ToString(FeeEstimateMode::SAT_VB)}; + const std::string incremental_fee{CFeeRate(DEFAULT_INCREMENTAL_RELAY_FEE).ToString(FeeEstimateMode::SAT_VB)}; return RPCHelpMan{method_name, "\nBumps the fee of an opt-in-RBF transaction T, replacing it with a new transaction B.\n" @@ -3416,7 +3408,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) {"conf_target", RPCArg::Type::NUM, /* default */ "wallet -txconfirmtarget", "Confirmation target in blocks\n"}, {"fee_rate", RPCArg::Type::AMOUNT, /* default */ "not set, fall back to wallet fee estimation", "\nSpecify a fee rate in " + CURRENCY_ATOM + "/vB instead of relying on the built-in fee estimator.\n" - "Must be at least " + incremental_fee + " " + CURRENCY_ATOM + "/vB higher than the current transaction fee rate.\n" + "Must be at least " + incremental_fee + " higher than the current transaction fee rate.\n" "WARNING: before version 0.21, fee_rate was in " + CURRENCY_UNIT + "/kvB. As of 0.21, fee_rate is in " + CURRENCY_ATOM + "/vB.\n"}, {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" "marked bip-125 replaceable. If true, the sequence numbers in the transaction will\n" @@ -3492,7 +3484,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) if (options.exists("replaceable")) { coin_control.m_signal_bip125_rbf = options["replaceable"].get_bool(); } - SetFeeEstimateMode(pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /* override_min_fee */ false); + SetFeeEstimateMode(*pwallet, coin_control, conf_target, options["estimate_mode"], options["fee_rate"], /* override_min_fee */ false); } // Make sure the results are valid at least up to the most recent block @@ -3604,7 +3596,7 @@ static RPCHelpMan rescanblockchain() } int start_height = 0; - Optional<int> stop_height = MakeOptional(false, int()); + Optional<int> stop_height; uint256 start_block; { LOCK(pwallet->cs_wallet); @@ -3653,7 +3645,7 @@ static RPCHelpMan rescanblockchain() }; } -class DescribeWalletAddressVisitor : public boost::static_visitor<UniValue> +class DescribeWalletAddressVisitor { public: const SigningProvider * const provider; @@ -3672,7 +3664,7 @@ public: UniValue subobj(UniValue::VOBJ); UniValue detail = DescribeAddress(embedded); subobj.pushKVs(detail); - UniValue wallet_detail = boost::apply_visitor(*this, embedded); + UniValue wallet_detail = std::visit(*this, embedded); subobj.pushKVs(wallet_detail); subobj.pushKV("address", EncodeDestination(embedded)); subobj.pushKV("scriptPubKey", HexStr(subscript)); @@ -3755,7 +3747,7 @@ static UniValue DescribeWalletAddress(const CWallet* const pwallet, const CTxDes provider = pwallet->GetSolvingProvider(script); } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider.get()), dest)); + ret.pushKVs(std::visit(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -4081,10 +4073,10 @@ static RPCHelpMan send() RPCExamples{"" "\nSend 0.1 BTC with a confirmation target of 6 blocks in economical fee estimate mode\n" + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.1}' 6 economical\n") + - "Send 0.2 BTC with a fee rate of 1 " + CURRENCY_ATOM + "/vB using positional arguments\n" - + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.2}' 0 \"\" 1\n") + + "Send 0.2 BTC with a fee rate of 1.1 " + CURRENCY_ATOM + "/vB using positional arguments\n" + + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.2}' null \"unset\" 1.1\n") + "Send 0.2 BTC with a fee rate of 1 " + CURRENCY_ATOM + "/vB using the options argument\n" - + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.2}' '{\"fee_rate\": 1}'\n") + + + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.2}' null \"unset\" null '{\"fee_rate\": 1}'\n") + "Send 0.3 BTC with a fee rate of 25 " + CURRENCY_ATOM + "/vB using named arguments\n" + HelpExampleCli("-named send", "outputs='{\"" + EXAMPLE_ADDRESS[0] + "\": 0.3}' fee_rate=25\n") + "Create a transaction that should confirm the next block, with a specific input, and return result without adding to wallet or broadcasting to the network\n" @@ -4096,7 +4088,7 @@ static RPCHelpMan send() UniValueType(), // outputs (ARR or OBJ, checked later) UniValue::VNUM, // conf_target UniValue::VSTR, // estimate_mode - UniValue::VNUM, // fee_rate + UniValueType(), // fee_rate, will be checked by AmountFromValue() in SetFeeEstimateMode() UniValue::VOBJ, // options }, true ); @@ -4241,7 +4233,7 @@ static RPCHelpMan sethdseed() // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set an HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); } EnsureWalletIsUnlocked(pwallet); @@ -4472,14 +4464,18 @@ static RPCHelpMan walletcreatefundedpsbt() static RPCHelpMan upgradewallet() { return RPCHelpMan{"upgradewallet", - "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified\n" + "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified.\n" "New keys may be generated and a new wallet backup will need to be made.", { - {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version"} + {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version."} }, RPCResult{ RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "wallet_name", "Name of wallet this operation was performed on"}, + {RPCResult::Type::NUM, "previous_version", "Version of wallet before this operation"}, + {RPCResult::Type::NUM, "current_version", "Version of wallet after this operation"}, + {RPCResult::Type::STR, "result", /* optional */ true, "Description of result, if no error"}, {RPCResult::Type::STR, "error", /* optional */ true, "Error message (if there is one)"} }, }, @@ -4502,11 +4498,27 @@ static RPCHelpMan upgradewallet() version = request.params[0].get_int(); } bilingual_str error; - if (!pwallet->UpgradeWallet(version, error)) { - throw JSONRPCError(RPC_WALLET_ERROR, error.original); + const int previous_version{pwallet->GetVersion()}; + const bool wallet_upgraded{pwallet->UpgradeWallet(version, error)}; + const int current_version{pwallet->GetVersion()}; + std::string result; + + if (wallet_upgraded) { + if (previous_version == current_version) { + result = "Already at latest version. Wallet version unchanged."; + } else { + result = strprintf("Wallet upgraded successfully from version %i to version %i.", previous_version, current_version); + } } + UniValue obj(UniValue::VOBJ); - if (!error.empty()) { + obj.pushKV("wallet_name", pwallet->GetName()); + obj.pushKV("previous_version", previous_version); + obj.pushKV("current_version", current_version); + if (!result.empty()) { + obj.pushKV("result", result); + } else { + CHECK_NONFATAL(!error.empty()); obj.pushKV("error", error.original); } return obj; diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 225b975067..09a9ec68cd 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -6,6 +6,7 @@ #include <fs.h> #include <streams.h> #include <util/translation.h> +#include <wallet/bdb.h> #include <wallet/salvage.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -27,11 +28,13 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v DatabaseStatus status; options.require_existing = true; options.verify = false; + options.require_format = DatabaseFormat::BERKELEY; std::unique_ptr<WalletDatabase> database = MakeDatabase(file_path, options, status, error); if (!database) return false; - std::string filename; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + BerkeleyDatabase& berkeley_database = static_cast<BerkeleyDatabase&>(*database); + std::string filename = berkeley_database.Filename(); + std::shared_ptr<BerkeleyEnvironment> env = berkeley_database.env; if (!env->Open(error)) { return false; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index d2e1be6402..15972fe7bb 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -3,12 +3,15 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key_io.h> +#include <logging.h> #include <outputtype.h> #include <script/descriptor.h> #include <script/sign.h> #include <util/bip32.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/system.h> +#include <util/time.h> #include <util/translation.h> #include <wallet/scriptpubkeyman.h> @@ -453,7 +456,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual hd_upgrade = true; } // Upgrade to HD chain split if necessary - if (IsFeatureSupported(new_version, FEATURE_HD_SPLIT)) { + if (!IsFeatureSupported(prev_version, FEATURE_HD_SPLIT) && IsFeatureSupported(new_version, FEATURE_HD_SPLIT)) { WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 3bf8f78120..8f6b69bc78 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -11,6 +11,7 @@ #include <script/standard.h> #include <util/error.h> #include <util/message.h> +#include <util/time.h> #include <wallet/crypter.h> #include <wallet/ismine.h> #include <wallet/walletdb.h> @@ -171,7 +172,7 @@ protected: WalletStorage& m_storage; public: - ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} + explicit ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} virtual ~ScriptPubKeyMan() {}; virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } @@ -303,7 +304,7 @@ private: /* the HD chain data model (external chain counters) */ CHDChain m_hd_chain; - std::unordered_map<CKeyID, CHDChain, KeyIDHasher> m_inactive_hd_chains; + std::unordered_map<CKeyID, CHDChain, SaltedSipHasher> m_inactive_hd_chains; /* HD derive new child key (on internal or external chain) */ void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); @@ -503,7 +504,7 @@ class LegacySigningProvider : public SigningProvider private: const LegacyScriptPubKeyMan& m_spk_man; public: - LegacySigningProvider(const LegacyScriptPubKeyMan& spk_man) : m_spk_man(spk_man) {} + explicit LegacySigningProvider(const LegacyScriptPubKeyMan& spk_man) : m_spk_man(spk_man) {} bool GetCScript(const CScriptID &scriptid, CScript& script) const override { return m_spk_man.GetCScript(scriptid, script); } bool HaveCScript(const CScriptID &scriptid) const override { return m_spk_man.HaveCScript(scriptid); } diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index d83332c194..0fb3b1d3c4 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -17,7 +17,6 @@ #include <sqlite3.h> #include <stdint.h> -static const char* const DATABASE_FILENAME = "wallet.dat"; static constexpr int32_t WALLET_SCHEMA_VERSION = 0; static Mutex g_sqlite_mutex; @@ -206,7 +205,9 @@ void SQLiteDatabase::Open() } if (m_db == nullptr) { - TryCreateDirectories(m_dir_path); + if (!m_mock) { + TryCreateDirectories(m_dir_path); + } int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr); if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret))); @@ -566,17 +567,11 @@ bool SQLiteBatch::TxnAbort() return res == SQLITE_OK; } -bool ExistsSQLiteDatabase(const fs::path& path) -{ - const fs::path file = path / DATABASE_FILENAME; - return fs::symlink_status(file).type() == fs::regular_file && IsSQLiteFile(file); -} - std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) { - const fs::path file = path / DATABASE_FILENAME; try { - auto db = MakeUnique<SQLiteDatabase>(path, file); + fs::path data_file = SQLiteDataFile(path); + auto db = MakeUnique<SQLiteDatabase>(data_file.parent_path(), data_file); if (options.verify && !db->Verify(error)) { status = DatabaseStatus::FAILED_VERIFY; return nullptr; @@ -594,37 +589,3 @@ std::string SQLiteDatabaseVersion() { return std::string(sqlite3_libversion()); } - -bool IsSQLiteFile(const fs::path& path) -{ - if (!fs::exists(path)) return false; - - // A SQLite Database file is at least 512 bytes. - boost::system::error_code ec; - auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); - if (size < 512) return false; - - fsbridge::ifstream file(path, std::ios::binary); - if (!file.is_open()) return false; - - // Magic is at beginning and is 16 bytes long - char magic[16]; - file.read(magic, 16); - - // Application id is at offset 68 and 4 bytes long - file.seekg(68, std::ios::beg); - char app_id[4]; - file.read(app_id, 4); - - file.close(); - - // Check the magic, see https://sqlite.org/fileformat2.html - std::string magic_str(magic, 16); - if (magic_str != std::string("SQLite format 3", 16)) { - return false; - } - - // Check the application id matches our network magic - return memcmp(Params().MessageStart(), app_id, 4) == 0; -} diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index 693a2ef55a..70ab4f797a 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -37,7 +37,7 @@ public: explicit SQLiteBatch(SQLiteDatabase& database); ~SQLiteBatch() override { Close(); } - /* No-op. See commeng on SQLiteDatabase::Flush */ + /* No-op. See comment on SQLiteDatabase::Flush */ void Flush() override {} void Close() override; @@ -113,10 +113,8 @@ public: sqlite3* m_db{nullptr}; }; -bool ExistsSQLiteDatabase(const fs::path& path); std::unique_ptr<SQLiteDatabase> MakeSQLiteDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); std::string SQLiteDatabaseVersion(); -bool IsSQLiteFile(const fs::path& path); #endif // BITCOIN_WALLET_SQLITE_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index f38ccba384..202804c9ff 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -64,7 +64,8 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo if (spendable) { CTxDestination dest; std::string error; - assert(wallet.GetNewDestination(OutputType::BECH32, "", dest, error)); + const bool destination_ok = wallet.GetNewDestination(OutputType::BECH32, "", dest, error); + assert(destination_ok); tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); } if (fIsFromMe) { @@ -283,7 +284,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Make sure that can use BnB when there are preset inputs empty_wallet(); { - std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); wallet->SetupLegacyScriptPubKeyMan(); @@ -452,7 +453,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount - // run the 'mtgox' test (see http://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) + // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change empty_wallet(); for (int j = 0; j < 20; j++) diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 8f0083cd2e..27179839b7 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,13 @@ BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) +static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, std::string& database_filename) +{ + fs::path data_file = BDBDataFile(path); + database_filename = data_file.filename().string(); + return GetBerkeleyEnv(data_file.parent_path()); +} + BOOST_AUTO_TEST_CASE(getwalletenv_file) { std::string test_name = "test_name.dat"; diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index c80310045a..aa7f1c83d8 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_wallet_client = MakeWalletClient(*m_chain, *Assert(m_node.args)); + m_wallet_client = MakeWalletClient(*m_node.chain, *Assert(m_node.args)); std::string sep; sep += fs::path::preferred_separator; diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index f5bade77df..37ae907de5 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,7 +19,6 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { fs::path m_datadir; fs::path m_cwd; std::map<std::string, fs::path> m_walletdir_path_cases; - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::WalletClient> m_wallet_client; }; diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 9b905569fc..e70b56c529 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index d5aed99d99..5d25885bd4 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,8 +27,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CKey uncompressedKey; uncompressedKey.MakeNewKey(false); CPubKey uncompressedPubkey = uncompressedKey.GetPubKey(); - NodeContext node; - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); + std::unique_ptr<interfaces::Chain>& chain = m_node.chain; CScript scriptPubKey; isminetype result; diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index f7c1337b0d..347a436429 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -17,9 +17,7 @@ BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(CanProvide) { // Set up wallet and keyman variables. - NodeContext node; - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 10ddfa22ef..5b421840e0 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 4d6f427618..badf2eb459 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -6,10 +6,10 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(m_chain.get(), "", CreateMockWalletDatabase()) + m_wallet(m_node.chain.get(), "", CreateMockWalletDatabase()) { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); - m_chain_notifications_handler = m_chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); + m_chain_notifications_handler = m_node.chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); m_wallet_client->registerRpcs(); } diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index ba8a5ff1f3..ab7fb8c42b 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -20,8 +20,7 @@ struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); - std::unique_ptr<interfaces::WalletClient> m_wallet_client = interfaces::MakeWalletClient(*m_chain, *Assert(m_node.args)); + std::unique_ptr<interfaces::WalletClient> m_wallet_client = interfaces::MakeWalletClient(*m_node.chain, *Assert(m_node.args)); CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 4911af08c6..5480f3ab22 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -28,6 +28,8 @@ RPCHelpMan importmulti(); RPCHelpMan dumpwallet(); RPCHelpMan importwallet(); +extern RecursiveMutex cs_wallets; + // Ensure that fee levels defined in the wallet are at least as high // as the default levels for node policy. static_assert(DEFAULT_TRANSACTION_MINFEE >= DEFAULT_MIN_RELAY_TX_FEE, "wallet minimum fee is smaller than default relay fee"); @@ -83,12 +85,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - NodeContext node; - auto chain = interfaces::MakeChain(node); - // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -107,7 +106,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -133,7 +132,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -158,7 +157,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -183,9 +182,6 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - NodeContext node; - auto chain = interfaces::MakeChain(node); - // Prune the older block file. { LOCK(cs_main); @@ -197,7 +193,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); @@ -255,14 +251,11 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - NodeContext node; - auto chain = interfaces::MakeChain(node); - std::string backup_file = (GetDataDir() / "wallet.backup").string(); // Import key into wallet and call dumpwallet to create backup file. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -284,7 +277,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -317,10 +310,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - NodeContext node; - auto chain = interfaces::MakeChain(node); - - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); @@ -495,7 +485,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique<CWallet>(m_chain.get(), "", CreateMockWalletDatabase()); + wallet = MakeUnique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -545,7 +535,6 @@ public: return it->second; } - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<CWallet> wallet; }; @@ -561,7 +550,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 1U); // Check initial balance from one mature coinbase transaction. @@ -577,7 +566,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 2U); // Lock both coins. Confirm number of available coins drops to 0. @@ -606,15 +595,13 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); - BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 2U); } BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - NodeContext node; - auto chain = interfaces::MakeChain(node); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -709,8 +696,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) { // Create new wallet with known key and unload it. - auto chain = interfaces::MakeChain(m_node); - auto wallet = TestLoadWallet(*chain); + auto wallet = TestLoadWallet(*m_node.chain); CKey key; key.MakeNewKey(true); AddKey(*wallet, key); @@ -745,12 +731,12 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); auto mempool_tx = TestSimpleSpend(*m_coinbase_txns[1], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); - BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + BOOST_CHECK(m_node.chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); // Reload wallet and make sure new transactions are detected despite events // being blocked - wallet = TestLoadWallet(*chain); + wallet = TestLoadWallet(*m_node.chain); BOOST_CHECK(rescan_completed); BOOST_CHECK_EQUAL(addtx_count, 2); { @@ -777,18 +763,20 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // deadlock during the sync and simulates a new block notification happening // as soon as possible. addtx_count = 0; - auto handler = HandleLoadWallet([&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet) { + auto handler = HandleLoadWallet([&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet, cs_wallets) { BOOST_CHECK(rescan_completed); m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); block_tx = TestSimpleSpend(*m_coinbase_txns[2], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); mempool_tx = TestSimpleSpend(*m_coinbase_txns[3], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); - BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + BOOST_CHECK(m_node.chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + LEAVE_CRITICAL_SECTION(cs_wallets); LEAVE_CRITICAL_SECTION(wallet->wallet()->cs_wallet); SyncWithValidationInterfaceQueue(); ENTER_CRITICAL_SECTION(wallet->wallet()->cs_wallet); + ENTER_CRITICAL_SECTION(cs_wallets); }); - wallet = TestLoadWallet(*chain); + wallet = TestLoadWallet(*m_node.chain); BOOST_CHECK_EQUAL(addtx_count, 4); { LOCK(wallet->cs_wallet); @@ -802,8 +790,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) { - auto chain = interfaces::MakeChain(m_node); - auto wallet = TestLoadWallet(*chain); + auto wallet = TestLoadWallet(*m_node.chain); CKey key; key.MakeNewKey(true); AddKey(*wallet, key); diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp new file mode 100644 index 0000000000..558121ae42 --- /dev/null +++ b/src/wallet/test/walletdb_tests.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2012-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/setup_common.h> +#include <clientversion.h> +#include <streams.h> +#include <uint256.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(walletdb_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) +{ + /** + * When ReadKeyValue() reads from either a "key" or "wkey" it first reads the CDataStream steam into a + * CPrivKey or CWalletKey respectively and then reads a hash of the pubkey and privkey into a uint256. + * Wallets from 0.8 or before do not store the pubkey/privkey hash, trying to read the hash from old + * wallets throws an exception, for backwards compatibility this read is wrapped in a try block to + * silently fail. The test here makes sure the type of exception thrown from CDataStream::read() + * matches the type we expect, otherwise we need to update the "key"/"wkey" exception type caught. + */ + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + uint256 dummy; + BOOST_CHECK_THROW(ssValue >> dummy, std::ios_base::failure); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index ff8bfff872..918946f9e7 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -52,7 +52,7 @@ const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ static const size_t OUTPUT_GROUP_MAX_ENTRIES = 10; -static RecursiveMutex cs_wallets; +RecursiveMutex cs_wallets; static std::vector<std::shared_ptr<CWallet>> vpwallets GUARDED_BY(cs_wallets); static std::list<LoadWalletFn> g_load_wallet_fns GUARDED_BY(cs_wallets); @@ -85,9 +85,9 @@ static void UpdateWalletSetting(interfaces::Chain& chain, std::vector<bilingual_str>& warnings) { if (load_on_startup == nullopt) return; - if (load_on_startup.get() && !AddWalletSetting(chain, wallet_name)) { + if (load_on_startup.value() && !AddWalletSetting(chain, wallet_name)) { warnings.emplace_back(Untranslated("Wallet load on startup setting could not be updated, so wallet may not be loaded next node startup.")); - } else if (!load_on_startup.get() && !RemoveWalletSetting(chain, wallet_name)) { + } else if (!load_on_startup.value() && !RemoveWalletSetting(chain, wallet_name)) { warnings.emplace_back(Untranslated("Wallet load on startup setting could not be updated, so wallet may still be loaded next node startup.")); } } @@ -419,7 +419,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; - WalletBatch(*database).WriteMasterKey(pMasterKey.first, pMasterKey.second); + WalletBatch(GetDatabase()).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) Lock(); return true; @@ -432,7 +432,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::chainStateFlushed(const CBlockLocator& loc) { - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); batch.WriteBestBlock(loc); } @@ -444,7 +444,7 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in) nWalletVersion = nVersion; { - WalletBatch* batch = batch_in ? batch_in : new WalletBatch(*database); + WalletBatch* batch = batch_in ? batch_in : new WalletBatch(GetDatabase()); if (nWalletVersion > 40000) batch->WriteMinVersion(nWalletVersion); if (!batch_in) @@ -484,12 +484,12 @@ bool CWallet::HasWalletSpend(const uint256& txid) const void CWallet::Flush() { - database->Flush(); + GetDatabase().Flush(); } void CWallet::Close() { - database->Close(); + GetDatabase().Close(); } void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> range) @@ -570,7 +570,7 @@ void CWallet::AddToSpends(const uint256& wtxid) { auto it = mapWallet.find(wtxid); assert(it != mapWallet.end()); - CWalletTx& thisTx = it->second; + const CWalletTx& thisTx = it->second; if (thisTx.IsCoinBase()) // Coinbases don't spend anything! return; @@ -615,7 +615,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - WalletBatch* encrypted_batch = new WalletBatch(*database); + WalletBatch* encrypted_batch = new WalletBatch(GetDatabase()); if (!encrypted_batch->TxnBegin()) { delete encrypted_batch; encrypted_batch = nullptr; @@ -667,12 +667,12 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Need to completely rewrite the wallet file; if we don't, bdb might keep // bits of the unencrypted private key in slack space in the database file. - database->Rewrite(); + GetDatabase().Rewrite(); // BDB seems to have a bad habit of writing old data into // slack space in .dat files; that is bad if the old data is // unencrypted private keys. So: - database->ReloadDbEnv(); + GetDatabase().ReloadDbEnv(); } NotifyStatusChanged(this); @@ -683,7 +683,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { LOCK(cs_wallet); - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); // Old wallets didn't have any defined order for transactions // Probably a bad idea to change the output of this @@ -744,7 +744,7 @@ int64_t CWallet::IncOrderPosNext(WalletBatch* batch) if (batch) { batch->WriteOrderPosNext(nOrderPosNext); } else { - WalletBatch(*database).WriteOrderPosNext(nOrderPosNext); + WalletBatch(GetDatabase()).WriteOrderPosNext(nOrderPosNext); } return nRet; } @@ -774,7 +774,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); bool success = true; if (!batch.WriteTx(wtx)) { @@ -846,7 +846,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio { LOCK(cs_wallet); - WalletBatch batch(*database, fFlushOnClose); + WalletBatch batch(GetDatabase(), fFlushOnClose); uint256 hash = tx->GetHash(); @@ -946,11 +946,12 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx } // If wallet doesn't have a chain (e.g wallet-tool), don't bother to update txn. if (HaveChain()) { - Optional<int> block_height = chain().getBlockHeight(wtx.m_confirm.hashBlock); - if (block_height) { + bool active; + int height; + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().inActiveChain(active).height(height)) && active) { // Update cached block height variable since it not stored in the // serialized transaction. - wtx.m_confirm.block_height = *block_height; + wtx.m_confirm.block_height = height; } else if (wtx.isConflicted() || wtx.isConfirmed()) { // If tx block (or conflicting block) was reorged out of chain // while the wallet was shutdown, change tx status to UNCONFIRMED @@ -1045,7 +1046,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK(cs_wallet); - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); std::set<uint256> todo; std::set<uint256> done; @@ -1053,7 +1054,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) // Can't mark abandoned if confirmed or in mempool auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); - CWalletTx& origtx = it->second; + const CWalletTx& origtx = it->second; if (origtx.GetDepthInMainChain() != 0 || origtx.InMempool()) { return false; } @@ -1108,7 +1109,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c return; // Do not flush the wallet here for performance reasons - WalletBatch batch(*database, false); + WalletBatch batch(GetDatabase(), false); std::set<uint256> todo; std::set<uint256> done; @@ -1177,9 +1178,8 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // Handle transactions that were removed from the mempool because they // conflict with transactions in a newly connected block. if (reason == MemPoolRemovalReason::CONFLICT) { - // Call SyncNotifications, so external -walletnotify notifications will - // be triggered for these transactions. Set Status::UNCONFIRMED instead - // of Status::CONFLICTED for a few reasons: + // Trigger external -walletnotify notifications for these transactions. + // Set Status::UNCONFIRMED instead of Status::CONFLICTED for a few reasons: // // 1. The transactionRemovedFromMempool callback does not currently // provide the conflicting block's hash and height, and for backwards @@ -1446,13 +1446,13 @@ void CWallet::SetWalletFlag(uint64_t flags) { LOCK(cs_wallet); m_wallet_flags |= flags; - if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) + if (!WalletBatch(GetDatabase()).WriteWalletFlags(m_wallet_flags)) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } void CWallet::UnsetWalletFlag(uint64_t flag) { - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); UnsetWalletFlagWithDB(batch, flag); } @@ -1491,7 +1491,7 @@ bool CWallet::AddWalletFlags(uint64_t flags) LOCK(cs_wallet); // We should never be writing unknown non-tolerable wallet flags assert(((flags & KNOWN_WALLET_FLAGS) >> 32) == (flags >> 32)); - if (!WalletBatch(*database).WriteWalletFlags(flags)) { + if (!WalletBatch(GetDatabase()).WriteWalletFlags(flags)) { throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } @@ -1582,7 +1582,7 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return false; } if (apply_label) { - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); for (const CScript& script : script_pub_keys) { CTxDestination dest; ExtractDestination(script, dest); @@ -1771,18 +1771,22 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } + // Read block data CBlock block; - bool next_block; + chain().findBlock(block_hash, FoundBlock().data(block)); + + // Find next block separately from reading data above, because reading + // is slow and there might be a reorg while it is read. + bool block_still_active = false; + bool next_block = false; uint256 next_block_hash; - bool reorg = false; - if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { + chain().findBlock(block_hash, FoundBlock().inActiveChain(block_still_active).nextBlock(FoundBlock().inActiveChain(next_block).hash(next_block_hash))); + + if (!block.IsNull()) { LOCK(cs_wallet); - next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); - if (reorg) { + if (!block_still_active) { // Abort scan if current block is no longer active, to prevent // marking transactions as coming from the wrong block. - // TODO: This should return success instead of failure, see - // https://github.com/bitcoin/bitcoin/pull/14711#issuecomment-458342518 result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; break; @@ -1797,13 +1801,12 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; - next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); } if (max_height && block_height >= *max_height) { break; } { - if (!next_block || reorg) { + if (!next_block) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg break; @@ -2706,7 +2709,7 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin return locktime; } -OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) +OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const { // If -changetype is specified, always use that change type. if (change_type) { @@ -2786,7 +2789,7 @@ bool CWallet::CreateTransactionInternal( CScript scriptChange; // coin control: send change to custom address - if (!boost::get<CNoDestination>(&coin_control.destChange)) { + if (!std::get_if<CNoDestination>(&coin_control.destChange)) { scriptChange = GetScriptForDestination(coin_control.destChange); } else { // no coin control: send change to newly generated address // Note: We use a new key here to keep it from being obvious which side is the change. @@ -3048,7 +3051,7 @@ bool CWallet::CreateTransactionInternal( // to avoid conflicting with other possible uses of nSequence, // and in the spirit of "smallest possible change from prior // behavior." - const uint32_t nSequence = coin_control.m_signal_bip125_rbf.get_value_or(m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : (CTxIn::SEQUENCE_FINAL - 1); + const uint32_t nSequence = coin_control.m_signal_bip125_rbf.value_or(m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : (CTxIn::SEQUENCE_FINAL - 1); for (const auto& coin : selected_coins) { txNew.vin.push_back(CTxIn(coin.outpoint, CScript(), nSequence)); } @@ -3109,13 +3112,14 @@ bool CWallet::CreateTransaction( bool sign) { int nChangePosIn = nChangePosInOut; - CTransactionRef tx2 = tx; + Assert(!tx); // tx is an out-param. TODO change the return type from bool to tx (or nullptr) bool res = CreateTransactionInternal(vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, fee_calc_out, sign); // try with avoidpartialspends unless it's enabled already if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; CAmount nFeeRet2; + CTransactionRef tx2; int nChangePosInOut2 = nChangePosIn; bilingual_str error2; // fired and forgotten; if an error occurs, we discard the results if (CreateTransactionInternal(vecSend, tx2, nFeeRet2, nChangePosInOut2, error2, tmp_cc, fee_calc_out, sign)) { @@ -3177,10 +3181,10 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) LOCK(cs_wallet); fFirstRunRet = false; - DBErrors nLoadWalletRet = WalletBatch(*database).LoadWallet(this); + DBErrors nLoadWalletRet = WalletBatch(GetDatabase()).LoadWallet(this); if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - if (database->Rewrite("\x04pool")) + if (GetDatabase().Rewrite("\x04pool")) { for (const auto& spk_man_pair : m_spk_managers) { spk_man_pair.second->RewriteDB(); @@ -3204,7 +3208,7 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { AssertLockHeld(cs_wallet); - DBErrors nZapSelectTxRet = WalletBatch(*database).ZapSelectTx(vHashIn, vHashOut); + DBErrors nZapSelectTxRet = WalletBatch(GetDatabase()).ZapSelectTx(vHashIn, vHashOut); for (const uint256& hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); @@ -3216,7 +3220,7 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 if (nZapSelectTxRet == DBErrors::NEED_REWRITE) { - if (database->Rewrite("\x04pool")) + if (GetDatabase().Rewrite("\x04pool")) { for (const auto& spk_man_pair : m_spk_managers) { spk_man_pair.second->RewriteDB(); @@ -3254,14 +3258,14 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose) { - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); return SetAddressBookWithDB(batch, address, strName, strPurpose); } bool CWallet::DelAddressBook(const CTxDestination& address) { bool is_mine; - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); { LOCK(cs_wallet); // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) @@ -3720,7 +3724,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key, const std::string &value) { - if (boost::get<CNoDestination>(&dest)) + if (std::get_if<CNoDestination>(&dest)) return false; m_address_book[dest].destdata.insert(std::make_pair(key, value)); @@ -4008,7 +4012,7 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain& chain, const std::st int rescan_height = 0; if (!gArgs.GetBoolArg("-rescan", false)) { - WalletBatch batch(*walletInstance->database); + WalletBatch batch(walletInstance->GetDatabase()); CBlockLocator locator; if (batch.ReadBestBlock(locator)) { if (const Optional<int> fork_height = chain.findLocatorFork(locator)) { @@ -4051,16 +4055,13 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain& chain, const std::st // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - // The way the 'time_first_key' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int64_t> time_first_key = MakeOptional(false, int64_t());; + Optional<int64_t> time_first_key; for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { int64_t time = spk_man->GetTimeFirstKey(); if (!time_first_key || time < *time_first_key) time_first_key = time; } if (time_first_key) { - if (Optional<int> first_block = chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { - rescan_height = *first_block; - } + chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, FoundBlock().height(rescan_height)); } { @@ -4071,7 +4072,7 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain& chain, const std::st } } walletInstance->chainStateFlushed(chain.getTipLocator()); - walletInstance->database->IncrementUpdateCounter(); + walletInstance->GetDatabase().IncrementUpdateCounter(); } { @@ -4111,9 +4112,8 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error) } else { WalletLogPrintf("Allowing wallet upgrade up to %i\n", version); } - if (version < prev_version) - { - error = _("Cannot downgrade wallet"); + if (version < prev_version) { + error = strprintf(_("Cannot downgrade wallet from version %i to version %i. Wallet version unchanged."), prev_version, version); return false; } @@ -4121,7 +4121,7 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error) // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT if (!CanSupportFeature(FEATURE_HD_SPLIT) && version >= FEATURE_HD_SPLIT && version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified."); + error = strprintf(_("Cannot upgrade a non HD split wallet from version %i to version %i without upgrading to support pre-split keypool. Please use version %i or no version specified."), prev_version, version, FEATURE_PRE_SPLIT_KEYPOOL); return false; } @@ -4150,7 +4150,7 @@ void CWallet::postInitProcess() bool CWallet::BackupWallet(const std::string& strDest) const { - return database->Backup(strDest); + return GetDatabase().Backup(strDest); } CKeyPool::CKeyPool() @@ -4453,7 +4453,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() void CWallet::AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) { - WalletBatch batch(*database); + WalletBatch batch(GetDatabase()); if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 69cf6b66a4..e3eae1dd95 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -695,7 +695,7 @@ private: std::string m_name; /** Internal database handle. */ - std::unique_ptr<WalletDatabase> database; + std::unique_ptr<WalletDatabase> const m_database; /** * The following is used to keep track of how far behind the wallet is @@ -729,14 +729,11 @@ public: */ mutable RecursiveMutex cs_wallet; - /** Get database handle used by this wallet. Ideally this function would - * not be necessary. - */ - WalletDatabase& GetDBHandle() + WalletDatabase& GetDatabase() const override { - return *database; + assert(static_cast<bool>(m_database)); + return *m_database; } - WalletDatabase& GetDatabase() const override { return *database; } /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -758,7 +755,7 @@ public: CWallet(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database) : m_chain(chain), m_name(name), - database(std::move(database)) + m_database(std::move(database)) { } @@ -933,7 +930,7 @@ public: Balance GetBalance(int min_depth = 0, bool avoid_reuse = true) const; CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; - OutputType TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend); + OutputType TransactionChangeType(const Optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const; /** * Insert additional inputs into the transaction by diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index aa3b3c10b0..4e6270220e 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -14,7 +14,9 @@ #include <util/system.h> #include <util/time.h> #include <util/translation.h> +#ifdef USE_BDB #include <wallet/bdb.h> +#endif #ifdef USE_SQLITE #include <wallet/sqlite.h> #endif @@ -362,7 +364,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, { ssValue >> hash; } - catch (...) {} + catch (const std::ios_base::failure&) {} bool fSkipCheck = false; @@ -424,7 +426,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, uint256 checksum; ssValue >> checksum; if ((checksum_valid = Hash(vchPrivKey) != checksum)) { - strErr = "Error reading wallet database: Crypted key corrupt"; + strErr = "Error reading wallet database: Encrypted key corrupt"; return false; } } @@ -943,7 +945,7 @@ void MaybeCompactWalletDB() } for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { - WalletDatabase& dbh = pwallet->GetDBHandle(); + WalletDatabase& dbh = pwallet->GetDatabase(); unsigned int nUpdateCounter = dbh.nUpdateCounter; @@ -1011,11 +1013,10 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas Optional<DatabaseFormat> format; if (exists) { - if (ExistsBerkeleyDatabase(path)) { + if (IsBDBFile(BDBDataFile(path))) { format = DatabaseFormat::BERKELEY; } -#ifdef USE_SQLITE - if (ExistsSQLiteDatabase(path)) { + if (IsSQLiteFile(SQLiteDataFile(path))) { if (format) { error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", path.string())); status = DatabaseStatus::FAILED_BAD_FORMAT; @@ -1023,7 +1024,6 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas } format = DatabaseFormat::SQLITE; } -#endif } else if (options.require_existing) { error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string())); status = DatabaseStatus::FAILED_NOT_FOUND; @@ -1052,15 +1052,31 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas // Format is not set when a db doesn't already exist, so use the format specified by the options if it is set. if (!format && options.require_format) format = options.require_format; + // If the format is not specified or detected, choose the default format based on what is available. We prefer BDB over SQLite for now. + if (!format) { #ifdef USE_SQLITE - if (format && format == DatabaseFormat::SQLITE) { - return MakeSQLiteDatabase(path, options, status, error); + format = DatabaseFormat::SQLITE; +#endif +#ifdef USE_BDB + format = DatabaseFormat::BERKELEY; +#endif } -#else - assert(format != DatabaseFormat::SQLITE); + + if (format == DatabaseFormat::SQLITE) { +#ifdef USE_SQLITE + return MakeSQLiteDatabase(path, options, status, error); #endif + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", path.string())); + status = DatabaseStatus::FAILED_BAD_FORMAT; + return nullptr; + } +#ifdef USE_BDB return MakeBerkeleyDatabase(path, options, status, error); +#endif + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", path.string())); + status = DatabaseStatus::FAILED_BAD_FORMAT; + return nullptr; } /** Return object for accessing dummy database with no read/write capabilities. */ @@ -1072,5 +1088,9 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() { +#ifdef USE_BDB return MakeUnique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); +#elif USE_SQLITE + return MakeUnique<SQLiteDatabase>("", "", true); +#endif } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 7f1b86e458..e7b2d7d780 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -8,7 +8,6 @@ #include <amount.h> #include <script/sign.h> -#include <wallet/bdb.h> #include <wallet/db.h> #include <wallet/walletutil.h> #include <key.h> diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 0e18d6a740..a1bb7343f4 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -5,6 +5,7 @@ #include <fs.h> #include <util/system.h> #include <util/translation.h> +#include <wallet/dump.h> #include <wallet/salvage.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -21,30 +22,27 @@ static void WalletToolReleaseWallet(CWallet* wallet) delete wallet; } -static void WalletCreate(CWallet* wallet_instance) +static void WalletCreate(CWallet* wallet_instance, uint64_t wallet_creation_flags) { LOCK(wallet_instance->cs_wallet); wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); + wallet_instance->AddWalletFlags(wallet_creation_flags); - // generate a new HD seed - auto spk_man = wallet_instance->GetOrCreateLegacyScriptPubKeyMan(); - CPubKey seed = spk_man->GenerateNewSeed(); - spk_man->SetHDSeed(seed); + if (!wallet_instance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + auto spk_man = wallet_instance->GetOrCreateLegacyScriptPubKeyMan(); + spk_man->SetupGeneration(false); + } else { + wallet_instance->SetupDescriptorScriptPubKeyMans(); + } tfm::format(std::cout, "Topping up keypool...\n"); wallet_instance->TopUpKeyPool(); } -static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, bool create) +static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, DatabaseOptions options) { - DatabaseOptions options; DatabaseStatus status; - if (create) { - options.require_create = true; - } else { - options.require_existing = true; - } bilingual_str error; std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error); if (!database) { @@ -85,7 +83,7 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa } } - if (create) WalletCreate(wallet_instance.get()); + if (options.require_create) WalletCreate(wallet_instance.get(), options.create_flags); return wallet_instance; } @@ -105,36 +103,85 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& name) { fs::path path = fs::absolute(name, GetWalletDir()); + if (args.IsArgSet("-format") && command != "createfromdump") { + tfm::format(std::cerr, "The -format option can only be used with the \"createfromdump\" command.\n"); + return false; + } + if (args.IsArgSet("-dumpfile") && command != "dump" && command != "createfromdump") { + tfm::format(std::cerr, "The -dumpfile option can only be used with the \"dump\" and \"createfromdump\" commands.\n"); + return false; + } + if (args.IsArgSet("-descriptors") && command != "create") { + tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); + return false; + } + if (command == "create") { - std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, /* create= */ true); + DatabaseOptions options; + options.require_create = true; + if (args.GetBoolArg("-descriptors", false)) { + options.create_flags |= WALLET_FLAG_DESCRIPTORS; + options.require_format = DatabaseFormat::SQLITE; + } + + std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); } - } else if (command == "info" || command == "salvage") { - if (command == "info") { - std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, /* create= */ false); - if (!wallet_instance) return false; - WalletShowInfo(wallet_instance.get()); - wallet_instance->Close(); - } else if (command == "salvage") { - bilingual_str error; - std::vector<bilingual_str> warnings; - bool ret = RecoverDatabaseFile(path, error, warnings); - if (!ret) { - for (const auto& warning : warnings) { - tfm::format(std::cerr, "%s\n", warning.original); - } - if (!error.empty()) { - tfm::format(std::cerr, "%s\n", error.original); - } + } else if (command == "info") { + DatabaseOptions options; + options.require_existing = true; + std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + if (!wallet_instance) return false; + WalletShowInfo(wallet_instance.get()); + wallet_instance->Close(); + } else if (command == "salvage") { +#ifdef USE_BDB + bilingual_str error; + std::vector<bilingual_str> warnings; + bool ret = RecoverDatabaseFile(path, error, warnings); + if (!ret) { + for (const auto& warning : warnings) { + tfm::format(std::cerr, "%s\n", warning.original); } + if (!error.empty()) { + tfm::format(std::cerr, "%s\n", error.original); + } + } + return ret; +#else + tfm::format(std::cerr, "Salvage command is not available as BDB support is not compiled"); + return false; +#endif + } else if (command == "dump") { + DatabaseOptions options; + options.require_existing = true; + std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + if (!wallet_instance) return false; + bilingual_str error; + bool ret = DumpWallet(*wallet_instance, error); + if (!ret && !error.empty()) { + tfm::format(std::cerr, "%s\n", error.original); return ret; } + tfm::format(std::cout, "The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n"); + return ret; + } else if (command == "createfromdump") { + bilingual_str error; + std::vector<bilingual_str> warnings; + bool ret = CreateFromDump(name, path, error, warnings); + for (const auto& warning : warnings) { + tfm::format(std::cout, "%s\n", warning.original); + } + if (!ret && !error.empty()) { + tfm::format(std::cerr, "%s\n", error.original); + } + return ret; } else { tfm::format(std::cerr, "Invalid command: %s\n", command); return false; diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index d0b8d6812a..f544a6f727 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ namespace WalletTool { void WalletShowInfo(CWallet* wallet_instance); -bool ExecuteWalletToolFunc(const std::string& command, const std::string& file); +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& file); } // namespace WalletTool diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 702293e6c7..dd2f071b6c 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,13 +7,6 @@ #include <logging.h> #include <util/system.h> -bool ExistsBerkeleyDatabase(const fs::path& path); -#ifdef USE_SQLITE -bool ExistsSQLiteDatabase(const fs::path& path); -#else -# define ExistsSQLiteDatabase(path) (false) -#endif - fs::path GetWalletDir() { fs::path path; @@ -36,50 +29,6 @@ fs::path GetWalletDir() return path; } -std::vector<fs::path> ListWalletDir() -{ - const fs::path wallet_dir = GetWalletDir(); - const size_t offset = wallet_dir.string().size() + 1; - std::vector<fs::path> paths; - boost::system::error_code ec; - - for (auto it = fs::recursive_directory_iterator(wallet_dir, ec); it != fs::recursive_directory_iterator(); it.increment(ec)) { - if (ec) { - LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); - continue; - } - - try { - // Get wallet path relative to walletdir by removing walletdir from the wallet path. - // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. - const fs::path path = it->path().string().substr(offset); - - if (it->status().type() == fs::directory_file && - (ExistsBerkeleyDatabase(it->path()) || ExistsSQLiteDatabase(it->path()))) { - // Found a directory which contains wallet.dat btree file, add it as a wallet. - paths.emplace_back(path); - } else if (it.level() == 0 && it->symlink_status().type() == fs::regular_file && ExistsBerkeleyDatabase(it->path())) { - if (it->path().filename() == "wallet.dat") { - // Found top-level wallet.dat btree file, add top level directory "" - // as a wallet. - paths.emplace_back(); - } else { - // Found top-level btree file not called wallet.dat. Current bitcoin - // software will never create these files but will allow them to be - // opened in a shared database environment for backwards compatibility. - // Add it to the list of available wallets. - paths.emplace_back(path); - } - } - } catch (const std::exception& e) { - LogPrintf("%s: Error scanning %s: %s\n", __func__, it->path().string(), e.what()); - it.no_push(); - } - } - - return paths; -} - bool IsFeatureSupported(int wallet_version, int feature_version) { return wallet_version >= feature_version; @@ -87,13 +36,9 @@ bool IsFeatureSupported(int wallet_version, int feature_version) WalletFeature GetClosestWalletFeature(int version) { - if (version >= FEATURE_LATEST) return FEATURE_LATEST; - if (version >= FEATURE_PRE_SPLIT_KEYPOOL) return FEATURE_PRE_SPLIT_KEYPOOL; - if (version >= FEATURE_NO_DEFAULT_KEY) return FEATURE_NO_DEFAULT_KEY; - if (version >= FEATURE_HD_SPLIT) return FEATURE_HD_SPLIT; - if (version >= FEATURE_HD) return FEATURE_HD; - if (version >= FEATURE_COMPRPUBKEY) return FEATURE_COMPRPUBKEY; - if (version >= FEATURE_WALLETCRYPT) return FEATURE_WALLETCRYPT; - if (version >= FEATURE_BASE) return FEATURE_BASE; + static constexpr std::array wallet_features{FEATURE_LATEST, FEATURE_PRE_SPLIT_KEYPOOL, FEATURE_NO_DEFAULT_KEY, FEATURE_HD_SPLIT, FEATURE_HD, FEATURE_COMPRPUBKEY, FEATURE_WALLETCRYPT, FEATURE_BASE}; + for (const WalletFeature& wf : wallet_features) { + if (version >= wf) return wf; + } return static_cast<WalletFeature>(0); } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 27521abd81..67b2ee2b98 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -65,9 +65,6 @@ enum WalletFlags : uint64_t { //! Get the path of the wallet directory. fs::path GetWalletDir(); -//! Get wallets in wallet directory. -std::vector<fs::path> ListWalletDir(); - /** Descriptor with some wallet metadata */ class WalletDescriptor { diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h index a55e02f2dc..660b0eed5d 100644 --- a/src/walletinitinterface.h +++ b/src/walletinitinterface.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/warnings.cpp b/src/warnings.cpp index 501bf7e637..60388cc713 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +14,6 @@ static Mutex g_warnings_mutex; static bilingual_str g_misc_warnings GUARDED_BY(g_warnings_mutex); -static bool fLargeWorkForkFound GUARDED_BY(g_warnings_mutex) = false; static bool fLargeWorkInvalidChainFound GUARDED_BY(g_warnings_mutex) = false; void SetMiscWarning(const bilingual_str& warning) @@ -23,18 +22,6 @@ void SetMiscWarning(const bilingual_str& warning) g_misc_warnings = warning; } -void SetfLargeWorkForkFound(bool flag) -{ - LOCK(g_warnings_mutex); - fLargeWorkForkFound = flag; -} - -bool GetfLargeWorkForkFound() -{ - LOCK(g_warnings_mutex); - return fLargeWorkForkFound; -} - void SetfLargeWorkInvalidChainFound(bool flag) { LOCK(g_warnings_mutex); @@ -60,10 +47,7 @@ bilingual_str GetWarnings(bool verbose) warnings_verbose.emplace_back(warnings_concise); } - if (fLargeWorkForkFound) { - warnings_concise = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); - warnings_verbose.emplace_back(warnings_concise); - } else if (fLargeWorkInvalidChainFound) { + if (fLargeWorkInvalidChainFound) { warnings_concise = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); warnings_verbose.emplace_back(warnings_concise); } diff --git a/src/warnings.h b/src/warnings.h index 28546eb753..c38edb4570 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,8 +11,6 @@ struct bilingual_str; void SetMiscWarning(const bilingual_str& warning); -void SetfLargeWorkForkFound(bool flag); -bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); /** Format a string that describes several potential problems detected by the core. * @param[in] verbose bool diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index 3938f6fd2c..90aefb0018 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index dddba8d6b6..6f0b202a18 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index a2f994d7df..86f47d71f3 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 788a383517..8f81bfd63f 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index f13ed6f537..c1d66bddb1 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/README.md b/test/README.md index 2341eef00d..17bf8a1406 100644 --- a/test/README.md +++ b/test/README.md @@ -264,7 +264,7 @@ Use the `-v` option for verbose output. | [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.781](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install mypy==0.781` | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.7.1](https://github.com/bitcoin/bitcoin/pull/19348) | [details...](https://github.com/koalaman/shellcheck#installing) | [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.17.1](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install codespell==1.17.1` +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [2.0.0](https://github.com/bitcoin/bitcoin/pull/20817) | `pip3 install codespell==2.0.0` Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. diff --git a/test/config.ini.in b/test/config.ini.in index 4b4a092a9d..77c9a720c3 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -17,6 +17,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py # Which components are enabled. These are commented out by `configure` if they were disabled when running config. @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @USE_SQLITE_TRUE@USE_SQLITE=true +@USE_BDB_TRUE@USE_BDB=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true @BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true diff --git a/test/functional/README.md b/test/functional/README.md index 82b30fed51..2d04413eb2 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -23,7 +23,7 @@ don't have test cases for. - The oldest supported Python version is specified in [doc/dependencies.md](/doc/dependencies.md). Consider using [pyenv](https://github.com/pyenv/pyenv), which checks [.python-version](/.python-version), to prevent accidentally introducing modern syntax from an unsupported Python version. - The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). + The CI linter job also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that could lead to bugs and issues in the test code. - Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability @@ -34,7 +34,7 @@ don't have test cases for. - When subclassing the BitcoinTestFramework, place overrides for the `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. -- Use `'{}'.format(x)` for string formatting, not `'%s' % x`. +- Use `f'{x}'` for string formatting in preference to `'{}'.format(x)` or `'%s' % x`. #### Naming guidelines diff --git a/test/functional/example_test.py b/test/functional/example_test.py index c28bb7115f..97f24e1b6e 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 60492350ee..6c5857c5ce 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 3cf7f3890c..158efb52c9 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -119,7 +119,7 @@ class FullBlockTest(BitcoinTestFramework): # Allow the block to mature blocks = [] for i in range(NUM_BUFFER_BLOCKS_TO_GENERATE): - blocks.append(self.next_block("maturitybuffer.{}".format(i))) + blocks.append(self.next_block(f"maturitybuffer.{i}")) self.save_spendable_output() self.send_blocks(blocks) @@ -151,8 +151,8 @@ class FullBlockTest(BitcoinTestFramework): if template.valid_in_block: continue - self.log.info("Reject block with invalid tx: %s", TxTemplate.__name__) - blockname = "for_invalid.%s" % TxTemplate.__name__ + self.log.info(f"Reject block with invalid tx: {TxTemplate.__name__}") + blockname = f"for_invalid.{TxTemplate.__name__}" badblock = self.next_block(blockname) badtx = template.get_tx() if TxTemplate != invalid_txs.InputMissing: @@ -1355,12 +1355,12 @@ class FullBlockTest(BitcoinTestFramework): # save the current tip so it can be spent by a later block def save_spendable_output(self): - self.log.debug("saving spendable output %s" % self.tip.vtx[0]) + self.log.debug(f"saving spendable output {self.tip.vtx[0]}") self.spendable_outputs.append(self.tip) # get an output that we previously marked as spendable def get_spendable_output(self): - self.log.debug("getting spendable output %s" % self.spendable_outputs[0].vtx[0]) + self.log.debug(f"getting spendable output {self.spendable_outputs[0].vtx[0]}") return self.spendable_outputs.pop(0).vtx[0] # move the tip back to a previous block diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 7a2e35c095..f9ece244fb 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 8a8a0c7614..8f522aee66 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -13,6 +13,7 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_greater_than_or_equal, + assert_raises_rpc_error, satoshi_round, ) @@ -262,6 +263,11 @@ class EstimateFeeTest(BitcoinTestFramework): self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) + self.log.info("Testing that fee estimation is disabled in blocksonly.") + self.restart_node(0, ["-blocksonly"]) + assert_raises_rpc_error(-32603, "Fee estimation disabled", + self.nodes[0].estimatesmartfee, 2) + if __name__ == '__main__': EstimateFeeTest().main() diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index 9f18413255..babe9bfc80 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify that starting bitcoin with -h works as expected.""" diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index abf87e8f0c..803feb509a 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for setting nMinimumChainWork on command line. diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 47bc8dbb49..f2313bac13 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index b0eac7056b..bdbfa5aed1 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -60,7 +60,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.wit_address = w0.getnewaddress(address_type='p2sh-segwit') self.wit_ms_address = wmulti.addmultisigaddress(1, [self.pubkey], '', 'p2sh-segwit')['address'] if not self.options.descriptors: - # Legacy wallets need to import these so that they are watched by the wallet. This is unnecssary (and does not need to be tested) for descriptor wallets + # Legacy wallets need to import these so that they are watched by the wallet. This is unnecessary (and does not need to be tested) for descriptor wallets wmulti.importaddress(self.ms_address) wmulti.importaddress(self.wit_ms_address) diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 05b658ed87..cd5eff9184 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 1b531ad51d..c6f55c62b4 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 31cea8d1b7..68585b7475 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with -reindex and -reindex-chainstate options. diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index a76e0f1b50..291df4c518 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind shutdown.""" diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 6e28cfb265..6ee2b72c11 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -444,6 +444,8 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= * standard: whether the (valid version of) spending is expected to be standard * err_msg: a string with an expected error message for failure (or None, if not cared about) * sigops_weight: the pre-taproot sigops weight consumed by a successful spend + * need_vin_vout_mismatch: whether this test requires being tested in a transaction input that has no corresponding + transaction output. """ conf = dict() @@ -1442,17 +1444,27 @@ class TaprootTest(BitcoinTestFramework): self.nodes[1].generate(101) self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - # Transfer funds to pre-taproot node. + # Re-connect nodes in case they have been disconnected + self.disconnect_nodes(0, 1) + self.connect_nodes(0, 1) + + # Transfer value of the largest 500 coins to pre-taproot node. addr = self.nodes[0].getnewaddress() + + unsp = self.nodes[1].listunspent() + unsp = sorted(unsp, key=lambda i: i['amount'], reverse=True) + unsp = unsp[:500] + rawtx = self.nodes[1].createrawtransaction( inputs=[{ 'txid': i['txid'], 'vout': i['vout'] - } for i in self.nodes[1].listunspent()], - outputs={addr: self.nodes[1].getbalance()}, + } for i in unsp], + outputs={addr: sum(i['amount'] for i in unsp)} ) rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] - # Transaction is too large to fit into the mempool, so put it into a block + + # Mine a block with the transaction block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) add_witness_commitment(block) block.rehash() diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 2e4f4796b0..062a7affb5 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index d675ae174c..9b2414cf2d 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 7168cb4ab2..eb08765ebf 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -16,20 +16,20 @@ Only v0.15.2 is required by this test. The rest is used in other backwards compa import os from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet class MempoolCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.wallet_names = [None, self.default_wallet_name] + self.wallet_names = [None] def skip_test_if_missing_module(self): - self.skip_if_no_wallet() self.skip_if_no_previous_releases() def setup_network(self): self.add_nodes(self.num_nodes, versions=[ - 150200, # oldest version supported by the test framework + 190100, # oldest version with getmempoolinfo.loaded (used to avoid intermittent issues) None, ]) self.start_nodes() @@ -38,8 +38,15 @@ class MempoolCompatibilityTest(BitcoinTestFramework): def run_test(self): self.log.info("Test that mempool.dat is compatible between versions") - old_node = self.nodes[0] - new_node = self.nodes[1] + old_node, new_node = self.nodes + new_wallet = MiniWallet(new_node) + new_wallet.generate(1) + new_node.generate(100) + # Sync the nodes to ensure old_node has the block that contains the coinbase that new_wallet will spend. + # Otherwise, because coinbases are only valid in a block and not as loose txns, if the nodes aren't synced + # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`. + self.connect_nodes(0, 1) + self.sync_blocks() recipient = old_node.getnewaddress() self.stop_node(1) @@ -58,7 +65,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): assert old_tx_hash in new_node.getrawmempool() self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") - unbroadcasted_tx_hash = new_node.sendtoaddress(recipient, 0.0001) + unbroadcasted_tx_hash = new_wallet.send_self_transfer(from_node=new_node)['txid'] assert unbroadcasted_tx_hash in new_node.getrawmempool() mempool = new_node.getrawmempool(True) assert mempool[unbroadcasted_tx_hash]['unbroadcast'] @@ -72,5 +79,6 @@ class MempoolCompatibilityTest(BitcoinTestFramework): assert old_tx_hash in old_node.getrawmempool() assert unbroadcasted_tx_hash in old_node.getrawmempool() + if __name__ == "__main__": MempoolCompatibilityTest().main() diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 8b9b7b155a..4c46075ae9 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -16,8 +16,8 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - find_vout_for_address, ) +from test_framework.wallet import MiniWallet DEFAULT_MEMPOOL_EXPIRY = 336 # hours CUSTOM_MEMPOOL_EXPIRY = 10 # hours @@ -26,44 +26,50 @@ CUSTOM_MEMPOOL_EXPIRY = 10 # hours class MempoolExpiryTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + self.setup_clean_chain = True def test_transaction_expiry(self, timeout): """Tests that a transaction expires after the expiry timeout and its children are removed as well.""" node = self.nodes[0] + self.wallet = MiniWallet(node) + + # Add enough mature utxos to the wallet so that all txs spend confirmed coins. + self.wallet.generate(4) + node.generate(100) # Send a parent transaction that will expire. - parent_address = node.getnewaddress() - parent_txid = node.sendtoaddress(parent_address, 1.0) + parent_txid = self.wallet.send_self_transfer(from_node=node)['txid'] + parent_utxo = self.wallet.get_utxo(txid=parent_txid) + independent_utxo = self.wallet.get_utxo() + + # Ensure the transactions we send to trigger the mempool check spend utxos that are independent of + # the transactions being tested for expiration. + trigger_utxo1 = self.wallet.get_utxo() + trigger_utxo2 = self.wallet.get_utxo() # Set the mocktime to the arrival time of the parent transaction. entry_time = node.getmempoolentry(parent_txid)['time'] node.setmocktime(entry_time) - # Create child transaction spending the parent transaction - vout = find_vout_for_address(node, parent_txid, parent_address) - inputs = [{'txid': parent_txid, 'vout': vout}] - outputs = {node.getnewaddress(): 0.99} - child_raw = node.createrawtransaction(inputs, outputs) - child_signed = node.signrawtransactionwithwallet(child_raw)['hex'] - - # Let half of the timeout elapse and broadcast the child transaction. + # Let half of the timeout elapse and broadcast the child transaction spending the parent transaction. half_expiry_time = entry_time + int(60 * 60 * timeout/2) node.setmocktime(half_expiry_time) - child_txid = node.sendrawtransaction(child_signed) + child_txid = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=parent_utxo)['txid'] + assert_equal(parent_txid, node.getmempoolentry(child_txid)['depends'][0]) self.log.info('Broadcast child transaction after {} hours.'.format( timedelta(seconds=(half_expiry_time-entry_time)))) + # Broadcast another (independent) transaction. + independent_txid = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=independent_utxo)['txid'] + # Let most of the timeout elapse and check that the parent tx is still # in the mempool. nearly_expiry_time = entry_time + 60 * 60 * timeout - 5 node.setmocktime(nearly_expiry_time) - # Expiry of mempool transactions is only checked when a new transaction - # is added to the to the mempool. - node.sendtoaddress(node.getnewaddress(), 1.0) + # Broadcast a transaction as the expiry of transactions in the mempool is only checked + # when a new transaction is added to the mempool. + self.wallet.send_self_transfer(from_node=node, utxo_to_spend=trigger_utxo1) self.log.info('Test parent tx not expired after {} hours.'.format( timedelta(seconds=(nearly_expiry_time-entry_time)))) assert_equal(entry_time, node.getmempoolentry(parent_txid)['time']) @@ -72,9 +78,8 @@ class MempoolExpiryTest(BitcoinTestFramework): # has passed. expiry_time = entry_time + 60 * 60 * timeout + 5 node.setmocktime(expiry_time) - # Expiry of mempool transactions is only checked when a new transaction - # is added to the to the mempool. - node.sendtoaddress(node.getnewaddress(), 1.0) + # Again, broadcast a transaction so the expiry of transactions in the mempool is checked. + self.wallet.send_self_transfer(from_node=node, utxo_to_spend=trigger_utxo2) self.log.info('Test parent tx expiry after {} hours.'.format( timedelta(seconds=(expiry_time-entry_time)))) assert_raises_rpc_error(-5, 'Transaction not in mempool', @@ -85,6 +90,11 @@ class MempoolExpiryTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Transaction not in mempool', node.getmempoolentry, child_txid) + # Check that the independent tx is still in the mempool. + self.log.info('Test the independent tx not expired after {} hours.'.format( + timedelta(seconds=(expiry_time-half_expiry_time)))) + assert_equal(half_expiry_time, node.getmempoolentry(independent_txid)['time']) + def run_test(self): self.log.info('Test default mempool expiry timeout of %d hours.' % DEFAULT_MEMPOOL_EXPIRY) diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index e956fe07d2..a9e2b000fb 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking carve-out allowing one final transaction in diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index d7cb7db9f8..461f9237ff 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 187c9026f6..4aa58270b6 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,69 +1,62 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" -from test_framework.blocktools import create_raw_transaction from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + self.setup_clean_chain = True def run_test(self): - node0_address = self.nodes[0].getnewaddress() + node = self.nodes[0] + wallet = MiniWallet(node) + + # Add enough mature utxos to the wallet so that all txs spend confirmed coins + wallet.generate(3) + node.generate(100) + # Spend block 1/2/3's coinbase transactions - # Mine a block. + # Mine a block # Create three more transactions, spending the spends - # Mine another block. + # Mine another block # ... make sure all the transactions are confirmed # Invalidate both blocks # ... make sure all the transactions are put back in the mempool # Mine a new block - # ... make sure all the transactions are confirmed again. - - b = [self.nodes[0].getblockhash(n) for n in range(1, 4)] - coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] - spends1_raw = [create_raw_transaction(self.nodes[0], txid, node0_address, amount=49.99) for txid in coinbase_txids] - spends1_id = [self.nodes[0].sendrawtransaction(tx) for tx in spends1_raw] - + # ... make sure all the transactions are confirmed again blocks = [] - blocks.extend(self.nodes[0].generate(1)) - - spends2_raw = [create_raw_transaction(self.nodes[0], txid, node0_address, amount=49.98) for txid in spends1_id] - spends2_id = [self.nodes[0].sendrawtransaction(tx) for tx in spends2_raw] + spends1_ids = [wallet.send_self_transfer(from_node=node)['txid'] for _ in range(3)] + blocks.extend(node.generate(1)) + spends2_ids = [wallet.send_self_transfer(from_node=node)['txid'] for _ in range(3)] + blocks.extend(node.generate(1)) - blocks.extend(self.nodes[0].generate(1)) + spends_ids = set(spends1_ids + spends2_ids) # mempool should be empty, all txns confirmed - assert_equal(set(self.nodes[0].getrawmempool()), set()) - for txid in spends1_id+spends2_id: - tx = self.nodes[0].gettransaction(txid) - assert tx["confirmations"] > 0 + assert_equal(set(node.getrawmempool()), set()) + confirmed_txns = set(node.getblock(blocks[0])['tx'] + node.getblock(blocks[1])['tx']) + # Checks that all spend txns are contained in the mined blocks + assert spends_ids < confirmed_txns # Use invalidateblock to re-org back - for node in self.nodes: - node.invalidateblock(blocks[0]) + node.invalidateblock(blocks[0]) # All txns should be back in mempool with 0 confirmations - assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) - for txid in spends1_id+spends2_id: - tx = self.nodes[0].gettransaction(txid) - assert tx["confirmations"] == 0 + assert_equal(set(node.getrawmempool()), spends_ids) # Generate another block, they should all get mined - self.nodes[0].generate(1) + blocks = node.generate(1) # mempool should be empty, all txns confirmed - assert_equal(set(self.nodes[0].getrawmempool()), set()) - for txid in spends1_id+spends2_id: - tx = self.nodes[0].gettransaction(txid) - assert tx["confirmations"] > 0 + assert_equal(set(node.getrawmempool()), set()) + confirmed_txns = set(node.getblock(blocks[0])['tx']) + assert spends_ids < confirmed_txns if __name__ == '__main__': diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index 854d506f0d..a249a73315 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test spending coinbase transactions. @@ -13,44 +13,48 @@ but less mature coinbase spends are NOT. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.blocktools import create_raw_transaction from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.wallet import MiniWallet class MempoolSpendCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + self.setup_clean_chain = True def run_test(self): + wallet = MiniWallet(self.nodes[0]) + + wallet.generate(200) chain_height = self.nodes[0].getblockcount() assert_equal(chain_height, 200) - node0_address = self.nodes[0].getnewaddress() # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is - # is too immature to spend. + # too immature to spend. b = [self.nodes[0].getblockhash(n) for n in range(101, 103)] coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] - spends_raw = [create_raw_transaction(self.nodes[0], txid, node0_address, amount=49.99) for txid in coinbase_txids] + utxo_101 = wallet.get_utxo(txid=coinbase_txids[0]) + utxo_102 = wallet.get_utxo(txid=coinbase_txids[1]) - spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0]) + spend_101_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_101)["txid"] # coinbase at height 102 should be too immature to spend - assert_raises_rpc_error(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1]) + assert_raises_rpc_error(-26, + "bad-txns-premature-spend-of-coinbase", + lambda: wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)) # mempool should have just spend_101: - assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ]) + assert_equal(self.nodes[0].getrawmempool(), [spend_101_id]) # mine a block, spend_101 should get confirmed self.nodes[0].generate(1) assert_equal(set(self.nodes[0].getrawmempool()), set()) # ... and now height 102 can be spent: - spend_102_id = self.nodes[0].sendrawtransaction(spends_raw[1]) - assert_equal(self.nodes[0].getrawmempool(), [ spend_102_id ]) + spend_102_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)["txid"] + assert_equal(self.nodes[0].getrawmempool(), [spend_102_id]) + if __name__ == '__main__': MempoolSpendCoinbaseTest().main() diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 2adafb1fdb..cc32f78e2e 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py new file mode 100755 index 0000000000..a63c3a3287 --- /dev/null +++ b/test/functional/p2p_add_connections.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test add_outbound_p2p_connection test framework functionality""" + +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +def check_node_connections(*, node, num_in, num_out): + info = node.getnetworkinfo() + assert_equal(info["connections_in"], num_in) + assert_equal(info["connections_out"], num_out) + + +class P2PAddConnections(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 2 + + def setup_network(self): + self.setup_nodes() + # Don't connect the nodes + + def run_test(self): + self.log.info("Add 8 outbounds to node 0") + for i in range(8): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="outbound-full-relay") + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # set p2p_idx based on the outbound connections already open to the + # node, so add 8 to account for the previous full-relay connections + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 8, connection_type="block-relay-only") + + self.log.info("Add 2 block-relay-only connections to node 1") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + self.nodes[1].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="block-relay-only") + + self.log.info("Add 5 inbound connections to node 1") + for i in range(5): + self.log.info(f"inbound: {i}") + self.nodes[1].add_p2p_connection(P2PInterface()) + + self.log.info("Add 8 outbounds to node 1") + for i in range(8): + self.log.info(f"outbound: {i}") + # bump p2p_idx to account for the 2 existing outbounds on node 1 + self.nodes[1].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 2) + + self.log.info("Check the connections opened as expected") + check_node_connections(node=self.nodes[0], num_in=0, num_out=10) + check_node_connections(node=self.nodes[1], num_in=5, num_out=10) + + self.log.info("Disconnect p2p connections & try to re-open") + self.nodes[0].disconnect_p2ps() + check_node_connections(node=self.nodes[0], num_in=0, num_out=0) + + self.log.info("Add 8 outbounds to node 0") + for i in range(8): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i) + check_node_connections(node=self.nodes[0], num_in=0, num_out=8) + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # bump p2p_idx to account for the 8 existing outbounds on node 0 + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 8, connection_type="block-relay-only") + check_node_connections(node=self.nodes[0], num_in=0, num_out=10) + + self.log.info("Restart node 0 and try to reconnect to p2ps") + self.restart_node(0) + + self.log.info("Add 4 outbounds to node 0") + for i in range(4): + self.log.info(f"outbound: {i}") + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i) + check_node_connections(node=self.nodes[0], num_in=0, num_out=4) + + self.log.info("Add 2 block-relay-only connections to node 0") + for i in range(2): + self.log.info(f"block-relay-only: {i}") + # bump p2p_idx to account for the 4 existing outbounds on node 0 + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i + 4, connection_type="block-relay-only") + check_node_connections(node=self.nodes[0], num_in=0, num_out=6) + + check_node_connections(node=self.nodes[1], num_in=5, num_out=10) + + +if __name__ == '__main__': + P2PAddConnections().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 80f262d0d3..91fbd722cf 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -19,8 +19,11 @@ from test_framework.util import ( ) import time +# Keep this with length <= 10. Addresses from larger messages are not relayed. ADDRS = [] -for i in range(10): +num_ipv4_addrs = 10 + +for i in range(num_ipv4_addrs): addr = CAddress() addr.time = int(time.time()) + i addr.nServices = NODE_NETWORK | NODE_WITNESS @@ -30,11 +33,15 @@ for i in range(10): class AddrReceiver(P2PInterface): + num_ipv4_received = 0 + def on_addr(self, message): for addr in message.addrs: assert_equal(addr.nServices, 9) + if not 8333 <= addr.port < 8343: + raise AssertionError("Invalid addr.port of {} (8333-8342 expected)".format(addr.port)) assert addr.ip.startswith('123.123.123.') - assert (8333 <= addr.port < 8343) + self.num_ipv4_received += 1 class AddrTest(BitcoinTestFramework): @@ -48,21 +55,33 @@ class AddrTest(BitcoinTestFramework): msg = msg_addr() self.log.info('Send too-large addr message') - msg.addrs = ADDRS * 101 + msg.addrs = ADDRS * 101 # more than 1000 addresses in one message with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) self.log.info('Check that addr message content is relayed and added to addrman') - addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) + num_receivers = 7 + receivers = [] + for _ in range(num_receivers): + receivers.append(self.nodes[0].add_p2p_connection(AddrReceiver())) msg.addrs = ADDRS - with self.nodes[0].assert_debug_log([ - 'Added 10 addresses from 127.0.0.1: 0 tried', + with self.nodes[0].assert_debug_log( + [ + 'Added {} addresses from 127.0.0.1: 0 tried'.format(num_ipv4_addrs), 'received: addr (301 bytes) peer=0', - 'sending addr (301 bytes) peer=1', - ]): + ] + ): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) - addr_receiver.sync_with_ping() + for receiver in receivers: + receiver.sync_with_ping() + + total_ipv4_received = sum(r.num_ipv4_received for r in receivers) + + # Every IPv4 address must be relayed to two peers, other than the + # originating node (addr_source). + ipv4_branching_factor = 2 + assert_equal(total_ipv4_received, num_ipv4_addrs * ipv4_branching_factor) if __name__ == '__main__': diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 3250cbecf9..03662babef 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_COMPACT_FILTERS (BIP 157/158). diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index e80422d1cf..c592ab52b1 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test p2p blocksonly""" +"""Test p2p blocksonly mode & block-relay-only connections.""" -from test_framework.messages import msg_tx, CTransaction, FromHex -from test_framework.p2p import P2PInterface +import time + +from test_framework.blocktools import create_transaction +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -16,50 +19,32 @@ class P2PBlocksOnly(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [["-blocksonly"]] + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + def run_test(self): - block_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) - - self.log.info('Check that txs from p2p are rejected and result in disconnect') - prevtx = self.nodes[0].getblock(self.nodes[0].getblockhash(1), 2)['tx'][0] - rawtx = self.nodes[0].createrawtransaction( - inputs=[{ - 'txid': prevtx['txid'], - 'vout': 0 - }], - outputs=[{ - self.nodes[0].get_deterministic_priv_key().address: 50 - 0.00125 - }], - ) - sigtx = self.nodes[0].signrawtransactionwithkey( - hexstring=rawtx, - privkeys=[self.nodes[0].get_deterministic_priv_key().key], - prevtxs=[{ - 'txid': prevtx['txid'], - 'vout': 0, - 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], - }], - )['hex'] + self.blocksonly_mode_tests() + self.blocks_relay_conn_tests() + + def blocksonly_mode_tests(self): + self.log.info("Tests with node running in -blocksonly mode") assert_equal(self.nodes[0].getnetworkinfo()['localrelay'], False) - with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): - block_relay_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) - block_relay_peer.wait_for_disconnect() - assert_equal(self.nodes[0].getmempoolinfo()['size'], 0) - # Remove the disconnected peer and add a new one. - del self.nodes[0].p2ps[0] - tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + self.nodes[0].add_p2p_connection(P2PInterface()) + tx, txid, tx_hex = self.check_p2p_tx_violation() self.log.info('Check that txs from rpc are not rejected and relayed to other peers') + tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], True) - txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + + assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(txid)]): - self.nodes[0].sendrawtransaction(sigtx) + self.nodes[0].sendrawtransaction(tx_hex) tx_relay_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) - self.log.info('Check that txs from peers with relay-permission are not rejected and relayed to others') self.log.info("Restarting node 0 with relay permission and blocksonly") - self.restart_node(0, ["-persistmempool=0", "-whitelist=relay@127.0.0.1", "-blocksonly", '-deprecatedrpc=whitelisted']) + self.restart_node(0, ["-persistmempool=0", "-whitelist=relay@127.0.0.1", "-blocksonly"]) assert_equal(self.nodes[0].getrawmempool(), []) first_peer = self.nodes[0].add_p2p_connection(P2PInterface()) second_peer = self.nodes[0].add_p2p_connection(P2PInterface()) @@ -67,8 +52,7 @@ class P2PBlocksOnly(BitcoinTestFramework): assert_equal(peer_1_info['permissions'], ['relay']) peer_2_info = self.nodes[0].getpeerinfo()[1] assert_equal(peer_2_info['permissions'], ['relay']) - assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) - txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) self.log.info('Check that the tx from first_peer with relay-permission is relayed to others (ie.second_peer)') with self.nodes[0].assert_debug_log(["received getdata"]): @@ -78,13 +62,58 @@ class P2PBlocksOnly(BitcoinTestFramework): # But if, for some reason, first_peer decides to relay transactions to us anyway, we should relay them to # second_peer since we gave relay permission to first_peer. # See https://github.com/bitcoin/bitcoin/issues/19943 for details. - first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) + first_peer.send_message(msg_tx(tx)) self.log.info('Check that the peer with relay-permission is still connected after sending the transaction') assert_equal(first_peer.is_connected, True) second_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) self.log.info("Relay-permission peer's transaction is accepted and relayed") + self.nodes[0].disconnect_p2ps() + self.nodes[0].generate(1) + + def blocks_relay_conn_tests(self): + self.log.info('Tests with node in normal mode with block-relay-only connections') + self.restart_node(0, ["-noblocksonly"]) # disables blocks only mode + assert_equal(self.nodes[0].getnetworkinfo()['localrelay'], True) + + # Ensure we disconnect if a block-relay-only connection sends us a transaction + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="block-relay-only") + assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], False) + _, txid, tx_hex = self.check_p2p_tx_violation(index=2) + + self.log.info("Check that txs from RPC are not sent to blockrelay connection") + conn = self.nodes[0].add_outbound_p2p_connection(P2PTxInvStore(), p2p_idx=1, connection_type="block-relay-only") + + self.nodes[0].sendrawtransaction(tx_hex) + + # Bump time forward to ensure nNextInvSend timer pops + self.nodes[0].setmocktime(int(time.time()) + 60) + + # Calling sync_with_ping twice requires that the node calls + # `ProcessMessage` twice, and thus ensures `SendMessages` must have + # been called at least once + conn.sync_with_ping() + conn.sync_with_ping() + assert(int(txid, 16) not in conn.get_invs()) + + def check_p2p_tx_violation(self, index=1): + self.log.info('Check that txs from P2P are rejected and result in disconnect') + input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(index), 2)['tx'][0]['txid'] + tx = create_transaction(self.nodes[0], input_txid, self.nodes[0].getnewaddress(), amount=(50 - 0.001)) + txid = tx.rehash() + tx_hex = tx.serialize().hex() + + with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): + self.nodes[0].p2ps[0].send_message(msg_tx(tx)) + self.nodes[0].p2ps[0].wait_for_disconnect() + assert_equal(self.nodes[0].getmempoolinfo()['size'], 0) + + # Remove the disconnected peer + del self.nodes[0].p2ps[0] + + return tx, txid, tx_hex + if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 611bffb25f..55573efc06 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -764,6 +764,34 @@ class CompactBlocksTest(BitcoinTestFramework): stalling_peer.send_and_ping(msg) assert_equal(int(node.getbestblockhash(), 16), block.sha256) + def test_highbandwidth_mode_states_via_getpeerinfo(self): + # create new p2p connection for a fresh state w/o any prior sendcmpct messages sent + hb_test_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) + + # assert the RPC getpeerinfo boolean fields `bip152_hb_{to, from}` + # match the given parameters for the last peer of a given node + def assert_highbandwidth_states(node, hb_to, hb_from): + peerinfo = node.getpeerinfo()[-1] + assert_equal(peerinfo['bip152_hb_to'], hb_to) + assert_equal(peerinfo['bip152_hb_from'], hb_from) + + # initially, neither node has selected the other peer as high-bandwidth yet + assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=False) + + # peer requests high-bandwidth mode by sending sendcmpct(1) + hb_test_node.send_and_ping(msg_sendcmpct(announce=True, version=2)) + assert_highbandwidth_states(self.nodes[0], hb_to=False, hb_from=True) + + # peer generates a block and sends it to node, which should + # select the peer as high-bandwidth (up to 3 peers according to BIP 152) + block = self.build_block_on_tip(self.nodes[0]) + hb_test_node.send_and_ping(msg_block(block)) + assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=True) + + # peer requests low-bandwidth mode by sending sendcmpct(0) + hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) + assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False) + def run_test(self): # Get the nodes out of IBD self.nodes[0].generate(1) @@ -822,6 +850,9 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Testing invalid index in cmpctblock message...") self.test_invalid_cmpctblock_message() + self.log.info("Testing high-bandwidth mode states via getpeerinfo...") + self.test_highbandwidth_mode_states_via_getpeerinfo() + if __name__ == '__main__': CompactBlocksTest().main() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index a7f51b5364..fb8529ad2b 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 72a255991c..d60aa5b383 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 51b7623fe3..a2a122b352 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index aaf862e6c8..469d66a851 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various fingerprinting protections. @@ -18,6 +18,7 @@ from test_framework.p2p import ( msg_block, msg_getdata, msg_getheaders, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -57,18 +58,6 @@ class P2PFingerprintTest(BitcoinTestFramework): msg.hashstop = block_hash node.send_message(msg) - # Check whether last block received from node has a given hash - def last_block_equals(self, expected_hash, node): - block_msg = node.last_message.get("block") - return block_msg and block_msg.block.rehash() == expected_hash - - # Check whether last block header received from node has a given hash - def last_header_equals(self, expected_hash, node): - headers_msg = node.last_message.get("headers") - return (headers_msg and - headers_msg.headers and - headers_msg.headers[0].rehash() == expected_hash) - # Checks that stale blocks timestamped more than a month ago are not served # by the node while recent stale blocks and old active chain blocks are. # This does not currently test that stale blocks timestamped within the @@ -101,34 +90,31 @@ class P2PFingerprintTest(BitcoinTestFramework): # Check that getdata request for stale block succeeds self.send_block_request(stale_hash, node0) - test_function = lambda: self.last_block_equals(stale_hash, node0) - self.wait_until(test_function, timeout=3) + node0.wait_for_block(stale_hash, timeout=3) # Check that getheader request for stale block header succeeds self.send_header_request(stale_hash, node0) - test_function = lambda: self.last_header_equals(stale_hash, node0) - self.wait_until(test_function, timeout=3) + node0.wait_for_header(hex(stale_hash), timeout=3) # Longest chain is extended so stale is much older than chain tip self.nodes[0].setmocktime(0) - tip = self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)[0] + block_hash = int(self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)[-1], 16) assert_equal(self.nodes[0].getblockcount(), 14) - - # Send getdata & getheaders to refresh last received getheader message - block_hash = int(tip, 16) - self.send_block_request(block_hash, node0) - self.send_header_request(block_hash, node0) - node0.sync_with_ping() + node0.wait_for_block(block_hash, timeout=3) # Request for very old stale block should now fail + with p2p_lock: + node0.last_message.pop("block", None) self.send_block_request(stale_hash, node0) - time.sleep(3) - assert not self.last_block_equals(stale_hash, node0) + node0.sync_with_ping() + assert "block" not in node0.last_message # Request for very old stale block header should now fail + with p2p_lock: + node0.last_message.pop("headers", None) self.send_header_request(stale_hash, node0) - time.sleep(3) - assert not self.last_header_equals(stale_hash, node0) + node0.sync_with_ping() + assert "headers" not in node0.last_message # Verify we can fetch very old blocks and headers on the active chain block_hash = int(block_hashes[2], 16) @@ -137,12 +123,11 @@ class P2PFingerprintTest(BitcoinTestFramework): node0.sync_with_ping() self.send_block_request(block_hash, node0) - test_function = lambda: self.last_block_equals(block_hash, node0) - self.wait_until(test_function, timeout=3) + node0.wait_for_block(block_hash, timeout=3) self.send_header_request(block_hash, node0) - test_function = lambda: self.last_header_equals(block_hash, node0) - self.wait_until(test_function, timeout=3) + node0.wait_for_header(hex(block_hash), timeout=3) + if __name__ == '__main__': P2PFingerprintTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index db72a361d9..c0b3c2cb12 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -18,6 +18,7 @@ from test_framework.messages import ( msg_inv, msg_ping, MSG_TX, + msg_version, ser_string, ) from test_framework.p2p import ( @@ -60,6 +61,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def run_test(self): self.test_buffer() + self.test_duplicate_version_msg() self.test_magic_bytes() self.test_checksum() self.test_size() @@ -92,6 +94,13 @@ class InvalidMessagesTest(BitcoinTestFramework): conn.sync_with_ping(timeout=1) self.nodes[0].disconnect_p2ps() + def test_duplicate_version_msg(self): + self.log.info("Test duplicate version message is ignored") + conn = self.nodes[0].add_p2p_connection(P2PDataStore()) + with self.nodes[0].assert_debug_log(['redundant version message from peer']): + conn.send_and_ping(msg_version()) + self.nodes[0].disconnect_p2ps() + def test_magic_bytes(self): self.log.info("Test message with invalid magic bytes disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 489d903c21..cca7390ae3 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -143,7 +143,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): } # Transactions that do not end up in the mempool # tx_orphan_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) - # tx_orphan_invaid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) + # tx_orphan_invalid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) self.wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 4b32d60db0..ca8bf908a9 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -24,8 +24,6 @@ from test_framework.util import ( assert_greater_than_or_equal, ) -DISCOURAGEMENT_THRESHOLD = 100 - class LazyPeer(P2PInterface): def __init__(self): @@ -93,27 +91,16 @@ class P2PLeakTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - # Peer that never sends a version. We will send a bunch of messages - # from this peer anyway and verify eventual disconnection. - no_version_disconnect_peer = self.nodes[0].add_p2p_connection( - LazyPeer(), send_version=False, wait_for_verack=False) - # Another peer that never sends a version, nor any other messages. It shouldn't receive anything from the node. no_version_idle_peer = self.nodes[0].add_p2p_connection(LazyPeer(), send_version=False, wait_for_verack=False) # Peer that sends a version but not a verack. no_verack_idle_peer = self.nodes[0].add_p2p_connection(NoVerackIdlePeer(), wait_for_verack=False) - # Send enough ping messages (any non-version message will do) prior to sending - # version to reach the peer discouragement threshold. This should get us disconnected. - for _ in range(DISCOURAGEMENT_THRESHOLD): - no_version_disconnect_peer.send_message(msg_ping()) - # Wait until we got the verack in response to the version. Though, don't wait for the node to receive the # verack, since we never sent one no_verack_idle_peer.wait_for_verack() - no_version_disconnect_peer.wait_until(lambda: no_version_disconnect_peer.ever_connected, check_connected=False) no_version_idle_peer.wait_until(lambda: no_version_idle_peer.ever_connected) no_verack_idle_peer.wait_until(lambda: no_verack_idle_peer.version_received) @@ -123,13 +110,9 @@ class P2PLeakTest(BitcoinTestFramework): #Give the node enough time to possibly leak out a message time.sleep(5) - # Expect this peer to be disconnected for misbehavior - assert not no_version_disconnect_peer.is_connected - self.nodes[0].disconnect_p2ps() # Make sure no unexpected messages came in - assert no_version_disconnect_peer.unexpected_msg == False assert no_version_idle_peer.unexpected_msg == False assert no_verack_idle_peer.unexpected_msg == False @@ -148,7 +131,7 @@ class P2PLeakTest(BitcoinTestFramework): p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 - with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): + with self.nodes[0].assert_debug_log(['peer=3 using obsolete version 31799; disconnecting']): p2p_old_peer.send_message(old_version_msg) p2p_old_peer.wait_for_disconnect() diff --git a/test/functional/p2p_nobloomfilter_messages.py b/test/functional/p2p_nobloomfilter_messages.py index c2311cb197..507a71b2a9 100755 --- a/test/functional/p2p_nobloomfilter_messages.py +++ b/test/functional/p2p_nobloomfilter_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test invalid p2p messages for nodes with bloom filters disabled. diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index b1a7ef6877..8d95f155c8 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_NETWORK_LIMITED. diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index ed82e6a2e2..62652d949d 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -38,35 +38,24 @@ class P2PPermissionsTests(BitcoinTestFramework): # default permissions (no specific permissions) ["-whitelist=127.0.0.1"], # Make sure the default values in the command line documentation match the ones here - ["relay", "noban", "mempool", "download"], - True) - - self.checkpermission( - # check without deprecatedrpc=whitelisted - ["-whitelist=127.0.0.1"], - # Make sure the default values in the command line documentation match the ones here - ["relay", "noban", "mempool", "download"], - None) + ["relay", "noban", "mempool", "download"]) self.checkpermission( # no permission (even with forcerelay) ["-whitelist=@127.0.0.1", "-whitelistforcerelay=1"], - [], - False) + []) self.checkpermission( # relay permission removed (no specific permissions) ["-whitelist=127.0.0.1", "-whitelistrelay=0"], - ["noban", "mempool", "download"], - True) + ["noban", "mempool", "download"]) self.checkpermission( # forcerelay and relay permission added # Legacy parameter interaction which set whitelistrelay to true # if whitelistforcerelay is true ["-whitelist=127.0.0.1", "-whitelistforcerelay"], - ["forcerelay", "relay", "noban", "mempool", "download"], - True) + ["forcerelay", "relay", "noban", "mempool", "download"]) # Let's make sure permissions are merged correctly # For this, we need to use whitebind instead of bind @@ -76,39 +65,28 @@ class P2PPermissionsTests(BitcoinTestFramework): self.checkpermission( ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay - ["noban", "bloomfilter", "forcerelay", "relay", "download"], - False) + ["noban", "bloomfilter", "forcerelay", "relay", "download"]) self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") self.checkpermission( # legacy whitelistrelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistrelay"], - ["noban", "mempool", "download"], - False) - - self.checkpermission( - # check without deprecatedrpc=whitelisted - ["-whitelist=noban,mempool@127.0.0.1", "-whitelistrelay"], - ["noban", "mempool", "download"], - None) + ["noban", "mempool", "download"]) self.checkpermission( # legacy whitelistforcerelay should be ignored ["-whitelist=noban,mempool@127.0.0.1", "-whitelistforcerelay"], - ["noban", "mempool", "download"], - False) + ["noban", "mempool", "download"]) self.checkpermission( # missing mempool permission to be considered legacy whitelisted ["-whitelist=noban@127.0.0.1"], - ["noban", "download"], - False) + ["noban", "download"]) self.checkpermission( # all permission added ["-whitelist=all@127.0.0.1"], - ["forcerelay", "noban", "mempool", "bloomfilter", "relay", "download", "addr"], - False) + ["forcerelay", "noban", "mempool", "bloomfilter", "relay", "download", "addr"]) self.stop_node(1) self.nodes[1].assert_start_raises_init_error(["-whitelist=oopsie@127.0.0.1"], "Invalid P2P permission", match=ErrorMatch.PARTIAL_REGEX) @@ -169,19 +147,13 @@ class P2PPermissionsTests(BitcoinTestFramework): reject_reason='Not relaying non-mempool transaction {} from forcerelay peer=0'.format(txid) ) - def checkpermission(self, args, expectedPermissions, whitelisted): - if whitelisted is not None: - args = [*args, '-deprecatedrpc=whitelisted'] + def checkpermission(self, args, expectedPermissions): self.restart_node(1, args) self.connect_nodes(0, 1) peerinfo = self.nodes[1].getpeerinfo()[0] - if whitelisted is None: - assert 'whitelisted' not in peerinfo - else: - assert_equal(peerinfo['whitelisted'], whitelisted) assert_equal(len(expectedPermissions), len(peerinfo['permissions'])) for p in expectedPermissions: - if not p in peerinfo['permissions']: + if p not in peerinfo['permissions']: raise AssertionError("Expected permissions %r is not granted." % p) def replaceinconfig(self, nodeid, old, new): diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index e99ecd8026..a9d8b12d70 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -37,7 +37,6 @@ from test_framework.messages import ( msg_tx, msg_block, msg_no_witness_tx, - msg_verack, ser_uint256, ser_vector, sha256, @@ -146,7 +145,7 @@ def test_witness_block(node, p2p, block, accepted, with_witness=True, reason=Non class TestP2PConn(P2PInterface): def __init__(self, wtxidrelay=False): - super().__init__() + super().__init__(wtxidrelay=wtxidrelay) self.getdataset = set() self.last_wtxidrelay = [] self.lastgetdata = [] @@ -157,13 +156,6 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): pass - def on_version(self, message): - if self.wtxidrelay: - super().on_version(message) - else: - self.send_message(msg_verack()) - self.nServices = message.nServices - def on_getdata(self, message): self.lastgetdata = message.inv for inv in message.inv: diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index ce12ce26ce..47832b04bf 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -57,8 +57,10 @@ class TimeoutsTest(BitcoinTestFramework): assert no_version_node.is_connected assert no_send_node.is_connected - no_verack_node.send_message(msg_ping()) - no_version_node.send_message(msg_ping()) + with self.nodes[0].assert_debug_log(['Unsupported message "ping" prior to verack from peer=0']): + no_verack_node.send_message(msg_ping()) + with self.nodes[0].assert_debug_log(['non-version message before version handshake. Message "ping" from peer=1']): + no_version_node.send_message(msg_ping()) sleep(1) diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 16d9302db8..8a751c6b54 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -30,8 +30,8 @@ import time class TestP2PConn(P2PInterface): - def __init__(self): - super().__init__() + def __init__(self, wtxidrelay=True): + super().__init__(wtxidrelay=wtxidrelay) self.tx_getdata_count = 0 def on_getdata(self, message): @@ -47,6 +47,7 @@ TXID_RELAY_DELAY = 2 # seconds OVERLOADED_PEER_DELAY = 2 # seconds MAX_GETDATA_IN_FLIGHT = 100 MAX_PEER_TX_ANNOUNCEMENTS = 5000 +NONPREF_PEER_TX_DELAY = 2 # Python test constants NUM_INBOUND = 10 @@ -168,8 +169,6 @@ class TxDownloadTest(BitcoinTestFramework): assert_equal(peer_fallback.tx_getdata_count, 0) self.nodes[0].setmocktime(int(time.time()) + GETDATA_TX_INTERVAL + 1) # Wait for request to peer_expiry to expire peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) self.restart_node(0) # reset mocktime def test_disconnect_fallback(self): @@ -187,8 +186,6 @@ class TxDownloadTest(BitcoinTestFramework): peer_disconnect.peer_disconnect() peer_disconnect.wait_for_disconnect() peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) def test_notfound_fallback(self): self.log.info('Check that notfounds will select another peer for download immediately') @@ -204,17 +201,42 @@ class TxDownloadTest(BitcoinTestFramework): assert_equal(peer_fallback.tx_getdata_count, 0) peer_notfound.send_and_ping(msg_notfound(vec=[CInv(MSG_WTX, WTXID)])) # Send notfound, so that fallback peer is selected peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) - def test_preferred_inv(self): - self.log.info('Check that invs from preferred peers are downloaded immediately') - self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + def test_preferred_inv(self, preferred=False): + if preferred: + self.log.info('Check invs from preferred peers are downloaded immediately') + self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + else: + self.log.info('Check invs from non-preferred peers are downloaded after {} s'.format(NONPREF_PEER_TX_DELAY)) + mock_time = int(time.time() + 1) + self.nodes[0].setmocktime(mock_time) peer = self.nodes[0].add_p2p_connection(TestP2PConn()) peer.send_message(msg_inv([CInv(t=MSG_WTX, h=0xff00ff00)])) - peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + peer.sync_with_ping() + if preferred: + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + else: + with p2p_lock: + assert_equal(peer.tx_getdata_count, 0) + self.nodes[0].setmocktime(mock_time + NONPREF_PEER_TX_DELAY) + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + + def test_txid_inv_delay(self, glob_wtxid=False): + self.log.info('Check that inv from a txid-relay peers are delayed by {} s, with a wtxid peer {}'.format(TXID_RELAY_DELAY, glob_wtxid)) + self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + mock_time = int(time.time() + 1) + self.nodes[0].setmocktime(mock_time) + peer = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=False)) + if glob_wtxid: + # Add a second wtxid-relay connection otherwise TXID_RELAY_DELAY is waived in + # lack of wtxid-relay peers + self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True)) + peer.send_message(msg_inv([CInv(t=MSG_TX, h=0xff11ff11)])) + peer.sync_with_ping() with p2p_lock: - assert_equal(peer.tx_getdata_count, 1) + assert_equal(peer.tx_getdata_count, 0 if glob_wtxid else 1) + self.nodes[0].setmocktime(mock_time + TXID_RELAY_DELAY) + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) def test_large_inv_batch(self): self.log.info('Test how large inv batches are handled with relay permission') @@ -229,8 +251,6 @@ class TxDownloadTest(BitcoinTestFramework): peer.send_message(msg_inv([CInv(t=MSG_WTX, h=wtxid) for wtxid in range(MAX_PEER_TX_ANNOUNCEMENTS + 1)])) peer.wait_until(lambda: peer.tx_getdata_count == MAX_PEER_TX_ANNOUNCEMENTS) peer.sync_with_ping() - with p2p_lock: - assert_equal(peer.tx_getdata_count, MAX_PEER_TX_ANNOUNCEMENTS) def test_spurious_notfound(self): self.log.info('Check that spurious notfound is ignored') @@ -242,6 +262,9 @@ class TxDownloadTest(BitcoinTestFramework): self.test_disconnect_fallback() self.test_notfound_fallback() self.test_preferred_inv() + self.test_preferred_inv(True) + self.test_txid_inv_delay() + self.test_txid_inv_delay(True) self.test_large_inv_batch() self.test_spurious_notfound() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index f965677408..99be6b7b8e 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -20,6 +20,7 @@ Tests correspond to code in rpc/blockchain.cpp. from decimal import Decimal import http.client +import os import subprocess from test_framework.blocktools import ( @@ -42,7 +43,9 @@ from test_framework.util import ( assert_raises_rpc_error, assert_is_hex_string, assert_is_hash_string, + get_datadir_path, ) +from test_framework.wallet import MiniWallet class BlockchainTest(BitcoinTestFramework): @@ -63,6 +66,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_getnetworkhashps() self._test_stopatheight() self._test_waitforblockheight() + self._test_getblock() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): @@ -364,6 +368,46 @@ class BlockchainTest(BitcoinTestFramework): assert_waitforheight(current_height) assert_waitforheight(current_height + 1) + def _test_getblock(self): + node = self.nodes[0] + + miniwallet = MiniWallet(node) + miniwallet.generate(5) + node.generate(100) + + fee_per_byte = Decimal('0.00000010') + fee_per_kb = 1000 * fee_per_byte + + miniwallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) + blockhash = node.generate(1)[0] + + self.log.info("Test that getblock with verbosity 1 doesn't include fee") + block = node.getblock(blockhash, 1) + assert 'fee' not in block['tx'][1] + + self.log.info('Test that getblock with verbosity 2 includes expected fee') + block = node.getblock(blockhash, 2) + tx = block['tx'][1] + assert 'fee' in tx + assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) + + self.log.info("Test that getblock with verbosity 2 still works with pruned Undo data") + datadir = get_datadir_path(self.options.tmpdir, 0) + + def move_block_file(old, new): + old_path = os.path.join(datadir, self.chain, 'blocks', old) + new_path = os.path.join(datadir, self.chain, 'blocks', new) + os.rename(old_path, new_path) + + # Move instead of deleting so we can restore chain state afterwards + move_block_file('rev00000.dat', 'rev_wrong') + + block = node.getblock(blockhash, 2) + assert 'fee' not in block['tx'][1] + + # Restore chain state + move_block_file('rev_wrong', 'rev00000.dat') + if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index f19c60dc36..31baeba582 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multisig RPCs""" diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index adcd8a7d4c..209d1182d7 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 3b76c7dd1e..81862ac69e 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -41,6 +41,8 @@ class EstimateFeeTest(BitcoinTestFramework): self.nodes[0].estimatesmartfee(1) # self.nodes[0].estimatesmartfee(1, None) self.nodes[0].estimatesmartfee(1, 'ECONOMICAL') + self.nodes[0].estimatesmartfee(1, 'unset') + self.nodes[0].estimatesmartfee(1, 'conservative') self.nodes[0].estimaterawfee(1) self.nodes[0].estimaterawfee(1, None) diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index f987867e61..569471dc87 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -717,15 +717,21 @@ class RawTransactionsTest(BitcoinTestFramework): result = node.fundrawtransaction(rawtx) # uses self.min_relay_tx_fee (set by settxfee) btc_kvb_to_sat_vb = 100000 # (1e5) - result1 = node.fundrawtransaction(rawtx, {"fee_rate": 2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}) + result1 = node.fundrawtransaction(rawtx, {"fee_rate": str(2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee)}) result2 = node.fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}) result3 = node.fundrawtransaction(rawtx, {"fee_rate": 10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}) - result4 = node.fundrawtransaction(rawtx, {"feeRate": 10 * self.min_relay_tx_fee}) + result4 = node.fundrawtransaction(rawtx, {"feeRate": str(10 * self.min_relay_tx_fee)}) + # Test that funding non-standard "zero-fee" transactions is valid. + result5 = self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 0}) + result6 = self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 0}) + result_fee_rate = result['fee'] * 1000 / count_bytes(result['hex']) assert_fee_amount(result1['fee'], count_bytes(result2['hex']), 2 * result_fee_rate) assert_fee_amount(result2['fee'], count_bytes(result2['hex']), 2 * result_fee_rate) assert_fee_amount(result3['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) assert_fee_amount(result4['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) + assert_fee_amount(result5['fee'], count_bytes(result5['hex']), 0) + assert_fee_amount(result6['fee'], count_bytes(result6['hex']), 0) # With no arguments passed, expect fee of 141 satoshis. assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) @@ -752,19 +758,15 @@ class RawTransactionsTest(BitcoinTestFramework): node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": n, "add_inputs": True}) self.log.info("Test invalid fee rate settings") - assert_raises_rpc_error(-8, "Invalid fee_rate 0.000 sat/vB (must be greater than 0)", - node.fundrawtransaction, rawtx, {"fee_rate": 0, "add_inputs": True}) - assert_raises_rpc_error(-8, "Invalid feeRate 0.00000000 BTC/kvB (must be greater than 0)", - node.fundrawtransaction, rawtx, {"feeRate": 0, "add_inputs": True}) for param, value in {("fee_rate", 100000), ("feeRate", 1.000)}: assert_raises_rpc_error(-4, "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)", node.fundrawtransaction, rawtx, {param: value, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount out of range", - node.fundrawtransaction, rawtx, {"fee_rate": -1, "add_inputs": True}) + node.fundrawtransaction, rawtx, {param: -1, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount is not a number or string", - node.fundrawtransaction, rawtx, {"fee_rate": {"foo": "bar"}, "add_inputs": True}) + node.fundrawtransaction, rawtx, {param: {"foo": "bar"}, "add_inputs": True}) assert_raises_rpc_error(-3, "Invalid amount", - node.fundrawtransaction, rawtx, {"fee_rate": "", "add_inputs": True}) + node.fundrawtransaction, rawtx, {param: "", "add_inputs": True}) self.log.info("Test min fee rate checks are bypassed with fundrawtxn, e.g. a fee_rate under 1 sat/vB is allowed") node.fundrawtransaction(rawtx, {"fee_rate": 0.99999999, "add_inputs": True}) diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py index c3c3622cf9..044dbd35bf 100755 --- a/test/functional/rpc_getblockfilter.py +++ b/test/functional/rpc_getblockfilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getblockfilter RPC.""" diff --git a/test/functional/rpc_getdescriptorinfo.py b/test/functional/rpc_getdescriptorinfo.py index ea064f9763..563f54c5e7 100755 --- a/test/functional/rpc_getdescriptorinfo.py +++ b/test/functional/rpc_getdescriptorinfo.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test getdescriptorinfo RPC. diff --git a/test/functional/rpc_getpeerinfo_deprecation.py b/test/functional/rpc_getpeerinfo_deprecation.py deleted file mode 100755 index 340a66e12f..0000000000 --- a/test/functional/rpc_getpeerinfo_deprecation.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test deprecation of getpeerinfo RPC fields.""" - -from test_framework.test_framework import BitcoinTestFramework - - -class GetpeerinfoDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.extra_args = [[], ["-deprecatedrpc=banscore"]] - - def run_test(self): - self.test_banscore_deprecation() - self.test_addnode_deprecation() - - def test_banscore_deprecation(self): - self.log.info("Test getpeerinfo by default no longer returns a banscore field") - assert "banscore" not in self.nodes[0].getpeerinfo()[0].keys() - - self.log.info("Test getpeerinfo returns banscore with -deprecatedrpc=banscore") - assert "banscore" in self.nodes[1].getpeerinfo()[0].keys() - - def test_addnode_deprecation(self): - self.restart_node(1, ["-deprecatedrpc=getpeerinfo_addnode"]) - self.connect_nodes(0, 1) - - self.log.info("Test getpeerinfo by default no longer returns an addnode field") - assert "addnode" not in self.nodes[0].getpeerinfo()[0].keys() - - self.log.info("Test getpeerinfo returns addnode with -deprecatedrpc=addnode") - assert "addnode" in self.nodes[1].getpeerinfo()[0].keys() - - -if __name__ == "__main__": - GetpeerinfoDeprecationTest().main() diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 9b981b864e..c83157a843 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC help output.""" diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index f884b8d293..114bd5e7e9 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 0493ceeb64..1398d1237f 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC misc output.""" diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 2efd090733..de0b7f303f 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 04d55b103f..7d3a5cc9ce 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 498197b5e5..b364077a9a 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -190,28 +190,29 @@ class PSBTTest(BitcoinTestFramework): self.log.info("Test walletcreatefundedpsbt fee rate of 10000 sat/vB and 0.1 BTC/kvB produces a total fee at or slightly below -maxtxfee (~0.05290000)") res1 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"fee_rate": 10000, "add_inputs": True}) assert_approx(res1["fee"], 0.055, 0.005) - res2 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"feeRate": 0.1, "add_inputs": True}) + res2 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"feeRate": "0.1", "add_inputs": True}) assert_approx(res2["fee"], 0.055, 0.005) + self.log.info("Test min fee rate checks with walletcreatefundedpsbt are bypassed, e.g. a fee_rate under 1 sat/vB is allowed") - res3 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"fee_rate": 0.99999999, "add_inputs": True}) + res3 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"fee_rate": "0.99999999", "add_inputs": True}) assert_approx(res3["fee"], 0.00000381, 0.0000001) res4 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"feeRate": 0.00000999, "add_inputs": True}) assert_approx(res4["fee"], 0.00000381, 0.0000001) + self.log.info("Test min fee rate checks with walletcreatefundedpsbt are bypassed and that funding non-standard 'zero-fee' transactions is valid") + for param in ["fee_rate", "feeRate"]: + assert_equal(self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {param: 0, "add_inputs": True})["fee"], 0) + self.log.info("Test invalid fee rate settings") - assert_raises_rpc_error(-8, "Invalid fee_rate 0.000 sat/vB (must be greater than 0)", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"fee_rate": 0, "add_inputs": True}) - assert_raises_rpc_error(-8, "Invalid feeRate 0.00000000 BTC/kvB (must be greater than 0)", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"feeRate": 0, "add_inputs": True}) for param, value in {("fee_rate", 100000), ("feeRate", 1)}: assert_raises_rpc_error(-4, "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)", self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: value, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount out of range", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"fee_rate": -1, "add_inputs": True}) + self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: -1, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount is not a number or string", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"fee_rate": {"foo": "bar"}, "add_inputs": True}) + self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: {"foo": "bar"}, "add_inputs": True}) assert_raises_rpc_error(-3, "Invalid amount", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"fee_rate": "", "add_inputs": True}) + self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: "", "add_inputs": True}) self.log.info("- raises RPC error if both feeRate and fee_rate are passed") assert_raises_rpc_error(-8, "Cannot specify both fee_rate (sat/vB) and feeRate (BTC/kvB)", diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 554c30c0d2..86c7b3fbcc 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -372,6 +372,13 @@ class RawTransactionsTest(BitcoinTestFramework): encrawtx = "01000000010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f505000000000000000000" decrawtx = self.nodes[0].decoderawtransaction(encrawtx, False) # decode as non-witness transaction assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + # known ambiguous transaction in the chain (see https://github.com/bitcoin/bitcoin/issues/20579) + encrawtx = "020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff4b03c68708046ff8415c622f4254432e434f4d2ffabe6d6de1965d02c68f928e5b244ab1965115a36f56eb997633c7f690124bbf43644e23080000000ca3d3af6d005a65ff0200fd00000000ffffffff03f4c1fb4b0000000016001497cfc76442fe717f2a3f0cc9c175f7561b6619970000000000000000266a24aa21a9ed957d1036a80343e0d1b659497e1b48a38ebe876a056d45965fac4a85cda84e1900000000000000002952534b424c4f434b3a8e092581ab01986cbadc84f4b43f4fa4bb9e7a2e2a0caf9b7cf64d939028e22c0120000000000000000000000000000000000000000000000000000000000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx) + decrawtx_wit = self.nodes[0].decoderawtransaction(encrawtx, True) + assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # fails to decode as non-witness transaction + assert_equal(decrawtx, decrawtx_wit) # the witness interpretation should be chosen + assert_equal(decrawtx['vin'][0]['coinbase'], "03c68708046ff8415c622f4254432e434f4d2ffabe6d6de1965d02c68f928e5b244ab1965115a36f56eb997633c7f690124bbf43644e23080000000ca3d3af6d005a65ff0200fd00000000") # Basic signrawtransaction test addr = self.nodes[1].getnewaddress() diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index bc48449084..fd5f8aa098 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the setban rpc call.""" @@ -15,6 +15,9 @@ class SetBanTests(BitcoinTestFramework): self.setup_clean_chain = True self.extra_args = [[],[]] + def is_banned(self, node, addr): + return any(e['address'] == addr for e in node.listbanned()) + def run_test(self): # Node 0 connects to Node 1, check that the noban permission is not granted self.connect_nodes(0, 1) @@ -42,5 +45,18 @@ class SetBanTests(BitcoinTestFramework): peerinfo = self.nodes[1].getpeerinfo()[0] assert(not 'noban' in peerinfo['permissions']) + self.log.info("Test that a non-IP address can be banned/unbanned") + node = self.nodes[1] + tor_addr = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion" + ip_addr = "1.2.3.4" + assert(not self.is_banned(node, tor_addr)) + assert(not self.is_banned(node, ip_addr)) + node.setban(tor_addr, "add") + assert(self.is_banned(node, tor_addr)) + assert(not self.is_banned(node, ip_addr)) + node.setban(tor_addr, "remove") + assert(not self.is_banned(self.nodes[1], tor_addr)) + assert(not self.is_banned(node, ip_addr)) + if __name__ == '__main__': SetBanTests().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index b962e1c3a5..2fbbdbbdf0 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -151,6 +151,19 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) assert not rawTxSigned['errors'][0]['witness'] + def test_fully_signed_tx(self): + self.log.info("Test signing a fully signed transaction does nothing") + self.nodes[0].walletpassphrase("password", 9999) + self.nodes[0].generate(101) + rawtx = self.nodes[0].createrawtransaction([], [{self.nodes[0].getnewaddress(): 10}]) + fundedtx = self.nodes[0].fundrawtransaction(rawtx) + signedtx = self.nodes[0].signrawtransactionwithwallet(fundedtx["hex"]) + assert_equal(signedtx["complete"], True) + signedtx2 = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) + assert_equal(signedtx2["complete"], True) + assert_equal(signedtx["hex"], signedtx2["hex"]) + self.nodes[0].walletlock() + def witness_script_test(self): self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") # Create a new P2SH-P2WSH 1-of-1 multisig address: @@ -231,6 +244,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.witness_script_test() self.OP_1NEGATE_test() self.test_with_lock_outputs() + self.test_fully_signed_tx() if __name__ == '__main__': diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 93fb62c5d6..528da0cbfc 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" @@ -78,7 +78,7 @@ class MerkleBlockTest(BitcoinTestFramework): # We can't get a proof if we specify transactions from different blocks assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[0].gettxoutproof, [txid1, txid3]) # Test empty list - assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, []) + assert_raises_rpc_error(-8, "Parameter 'txids' cannot be empty", self.nodes[0].gettxoutproof, []) # Test duplicate txid assert_raises_rpc_error(-8, 'Invalid parameter, duplicated txid', self.nodes[0].gettxoutproof, [txid1, txid1]) diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index daf02fc4f3..7cedb4336b 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiple RPC users.""" @@ -99,11 +99,18 @@ class HTTPBasicsTest(BitcoinTestFramework): self.test_auth(self.nodes[1], self.rpcuser, self.rpcpassword) - self.log.info('Check that failure to write cookie file will abort the node gracefully') + init_error = 'Error: Unable to start HTTP server. See debug log for details.' + + self.log.info('Check -rpcauth are validated') + # Empty -rpcauth= are ignored + self.restart_node(0, extra_args=['-rpcauth=']) self.stop_node(0) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo']) + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error, extra_args=['-rpcauth=foo:bar']) + + self.log.info('Check that failure to write cookie file will abort the node gracefully') cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') os.mkdir(cookie_file) - init_error = 'Error: Unable to start HTTP server. See debug log for details.' self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 6b7214f03a..e691b63df6 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index f3d13c049b..e0cbab45ce 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -27,7 +27,7 @@ def xor_bytes(b0, b1): def jacobi_symbol(n, k): """Compute the Jacobi symbol of n modulo k - See http://en.wikipedia.org/wiki/Jacobi_symbol + See https://en.wikipedia.org/wiki/Jacobi_symbol For our application k is always prime, so this is the same as the Legendre symbol.""" assert k > 0 and k & 1, "jacobi symbol is only defined for positive odd k" diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index ff7f73bdf4..bab4ad0008 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -51,7 +51,6 @@ MAX_HEADERS_RESULTS = 2000 # Number of headers sent in one getheaders result MAX_INV_SIZE = 50000 # Maximum number of entries in an 'inv' protocol message NODE_NETWORK = (1 << 0) -NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) NODE_COMPACT_FILTERS = (1 << 6) diff --git a/test/functional/test_framework/muhash.py b/test/functional/test_framework/muhash.py index 97d02359cb..183548f71f 100644 --- a/test/functional/test_framework/muhash.py +++ b/test/functional/test_framework/muhash.py @@ -78,11 +78,13 @@ class MuHash3072: def insert(self, data): """Insert a byte array data in the set.""" - self.numerator = (self.numerator * data_to_num3072(data)) % self.MODULUS + data_hash = hashlib.sha256(data).digest() + self.numerator = (self.numerator * data_to_num3072(data_hash)) % self.MODULUS def remove(self, data): """Remove a byte array from the set.""" - self.denominator = (self.denominator * data_to_num3072(data)) % self.MODULUS + data_hash = hashlib.sha256(data).digest() + self.denominator = (self.denominator * data_to_num3072(data_hash)) % self.MODULUS def digest(self): """Extract the final hash. Does not modify this object.""" @@ -93,12 +95,12 @@ class MuHash3072: class TestFrameworkMuhash(unittest.TestCase): def test_muhash(self): muhash = MuHash3072() - muhash.insert([0]*32) - muhash.insert([1] + [0]*31) - muhash.remove([2] + [0]*31) + muhash.insert(b'\x00' * 32) + muhash.insert((b'\x01' + b'\x00' * 31)) + muhash.remove((b'\x02' + b'\x00' * 31)) finalized = muhash.digest() # This mirrors the result in the C++ MuHash3072 unit test - self.assertEqual(finalized[::-1].hex(), "a44e16d5e34d259b349af21c06e65d653915d2e208e4e03f389af750dc0bfdc3") + self.assertEqual(finalized[::-1].hex(), "10d312b100cbd32ada024a6646e40d3482fcff103668d2625f10002a607d5863") def test_chacha20(self): def chacha_check(key, result): diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index c98424e8e2..e047e7fa14 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -84,7 +84,7 @@ def get_bind_addrs(pid): bind_addrs.append(conn[1]) return bind_addrs -# from: http://code.activestate.com/recipes/439093/ +# from: https://code.activestate.com/recipes/439093/ def all_interfaces(): ''' Return all interfaces that are up diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 6846d31221..fa4a567aac 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -71,7 +71,11 @@ from test_framework.messages import ( NODE_WITNESS, sha256, ) -from test_framework.util import wait_until_helper +from test_framework.util import ( + MAX_NODES, + p2p_port, + wait_until_helper, +) logger = logging.getLogger("TestFramework.p2p") @@ -139,7 +143,7 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): + def peer_connect_helper(self, dstaddr, dstport, net, timeout_factor): assert not self.is_connected self.timeout_factor = timeout_factor self.dstaddr = dstaddr @@ -148,12 +152,20 @@ class P2PConnection(asyncio.Protocol): self.on_connection_send_msg = None self.recvbuf = b"" self.magic_bytes = MAGIC_BYTES[net] - logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) + + def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): + self.peer_connect_helper(dstaddr, dstport, net, timeout_factor) loop = NetworkThread.network_event_loop - conn_gen_unsafe = loop.create_connection(lambda: self, host=self.dstaddr, port=self.dstport) - conn_gen = lambda: loop.call_soon_threadsafe(loop.create_task, conn_gen_unsafe) - return conn_gen + logger.debug('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) + coroutine = loop.create_connection(lambda: self, host=self.dstaddr, port=self.dstport) + return lambda: loop.call_soon_threadsafe(loop.create_task, coroutine) + + def peer_accept_connection(self, connect_id, connect_cb=lambda: None, *, net, timeout_factor): + self.peer_connect_helper('0', 0, net, timeout_factor) + + logger.debug('Listening for Bitcoin Node with id: {}'.format(connect_id)) + return lambda: NetworkThread.listen(self, connect_cb, idx=connect_id) def peer_disconnect(self): # Connection could have already been closed by other end. @@ -289,7 +301,7 @@ class P2PInterface(P2PConnection): Individual testcases should subclass this and override the on_* methods if they want to alter message handling behaviour.""" - def __init__(self, support_addrv2=False): + def __init__(self, support_addrv2=False, wtxidrelay=True): super().__init__() # Track number of messages of each type received. @@ -309,18 +321,30 @@ class P2PInterface(P2PConnection): self.support_addrv2 = support_addrv2 - def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs): + # If the peer supports wtxid-relay + self.wtxidrelay = wtxidrelay + + def peer_connect_send_version(self, services): + # Send a version msg + vt = msg_version() + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.on_connection_send_msg = vt # Will be sent in connection_made callback + + def peer_connect(self, *args, services=NODE_NETWORK | NODE_WITNESS, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) if send_version: - # Send a version msg - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.on_connection_send_msg = vt # Will be sent soon after connection_made + self.peer_connect_send_version(services) + + return create_conn + + def peer_accept_connection(self, *args, services=NODE_NETWORK | NODE_WITNESS, **kwargs): + create_conn = super().peer_accept_connection(*args, **kwargs) + self.peer_connect_send_version(services) return create_conn @@ -394,11 +418,11 @@ class P2PInterface(P2PConnection): def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) - if message.nVersion >= 70016: + if message.nVersion >= 70016 and self.wtxidrelay: self.send_message(msg_wtxidrelay()) - self.send_message(msg_verack()) if self.support_addrv2: self.send_message(msg_sendaddrv2()) + self.send_message(msg_verack()) self.nServices = message.nServices # Connection helper methods @@ -411,6 +435,10 @@ class P2PInterface(P2PConnection): wait_until_helper(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) + def wait_for_connect(self, timeout=60): + test_function = lambda: self.is_connected + wait_until_helper(test_function, timeout=timeout, lock=p2p_lock) + def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected self.wait_until(test_function, timeout=timeout, check_connected=False) @@ -524,6 +552,8 @@ class NetworkThread(threading.Thread): # There is only one event loop and no more than one thread must be created assert not self.network_event_loop + NetworkThread.listeners = {} + NetworkThread.protos = {} NetworkThread.network_event_loop = asyncio.new_event_loop() def run(self): @@ -539,6 +569,48 @@ class NetworkThread(threading.Thread): # Safe to remove event loop. NetworkThread.network_event_loop = None + @classmethod + def listen(cls, p2p, callback, port=None, addr=None, idx=1): + """ Ensure a listening server is running on the given port, and run the + protocol specified by `p2p` on the next connection to it. Once ready + for connections, call `callback`.""" + + if port is None: + assert 0 < idx <= MAX_NODES + port = p2p_port(MAX_NODES - idx) + if addr is None: + addr = '127.0.0.1' + + coroutine = cls.create_listen_server(addr, port, callback, p2p) + cls.network_event_loop.call_soon_threadsafe(cls.network_event_loop.create_task, coroutine) + + @classmethod + async def create_listen_server(cls, addr, port, callback, proto): + def peer_protocol(): + """Returns a function that does the protocol handling for a new + connection. To allow different connections to have different + behaviors, the protocol function is first put in the cls.protos + dict. When the connection is made, the function removes the + protocol function from that dict, and returns it so the event loop + can start executing it.""" + response = cls.protos.get((addr, port)) + cls.protos[(addr, port)] = None + return response + + if (addr, port) not in cls.listeners: + # When creating a listener on a given (addr, port) we only need to + # do it once. If we want different behaviors for different + # connections, we can accomplish this by providing different + # `proto` functions + + listener = await cls.network_event_loop.create_server(peer_protocol, addr, port) + logger.debug("Listening server on %s:%d should be started" % (addr, port)) + cls.listeners[(addr, port)] = listener + + cls.protos[(addr, port)] = proto + callback(addr, port) + + class P2PDataStore(P2PInterface): """A P2P data store class. diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 8e5848d493..be0e9f24e2 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -787,7 +787,7 @@ def TaprootSignatureHash(txTo, spent_utxos, hash_type, input_index = 0, scriptpa def taproot_tree_helper(scripts): if len(scripts) == 0: - return ([], bytes(0 for _ in range(32))) + return ([], bytes()) if len(scripts) == 1: # One entry: treat as a leaf script = scripts[0] @@ -824,21 +824,33 @@ def taproot_tree_helper(scripts): h = TaggedHash("TapBranch", left_h + right_h) return (left + right, h) +# A TaprootInfo object has the following fields: +# - scriptPubKey: the scriptPubKey (witness v1 CScript) +# - inner_pubkey: the inner pubkey (32 bytes) +# - negflag: whether the pubkey in the scriptPubKey was negated from inner_pubkey+tweak*G (bool). +# - tweak: the tweak (32 bytes) +# - leaves: a dict of name -> TaprootLeafInfo objects for all known leaves TaprootInfo = namedtuple("TaprootInfo", "scriptPubKey,inner_pubkey,negflag,tweak,leaves") + +# A TaprootLeafInfo object has the following fields: +# - script: the leaf script (CScript or bytes) +# - version: the leaf version (0xc0 for BIP342 tapscript) +# - merklebranch: the merkle branch to use for this leaf (32*N bytes) TaprootLeafInfo = namedtuple("TaprootLeafInfo", "script,version,merklebranch") def taproot_construct(pubkey, scripts=None): """Construct a tree of Taproot spending conditions - pubkey: an ECPubKey object for the internal pubkey + pubkey: a 32-byte xonly pubkey for the internal pubkey (bytes) scripts: a list of items; each item is either: - - a (name, CScript) tuple - - a (name, CScript, leaf version) tuple + - a (name, CScript or bytes, leaf version) tuple + - a (name, CScript or bytes) tuple (defaulting to leaf version 0xc0) - another list of items (with the same structure) - - a function, which specifies how to compute the hashing partner - in function of the hash of whatever it is combined with + - a list of two items; the first of which is an item itself, and the + second is a function. The function takes as input the Merkle root of the + first item, and produces a (fictitious) partner to hash with. - Returns: script (sPK or redeemScript), tweak, {name:(script, leaf version, negation flag, innerkey, merklepath), ...} + Returns: a TaprootInfo object """ if scripts is None: scripts = [] diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 318a438705..457be6b0e6 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 831599913d..7c5317480c 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -332,7 +332,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Methods to override in subclass test scripts. def set_test_params(self): - """Tests must this method to change default values for number of nodes, topology, etc""" + """Tests must override this method to change default values for number of nodes, topology, etc""" raise NotImplementedError def add_options(self, parser): @@ -517,13 +517,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def stop_node(self, i, expected_stderr='', wait=0): """Stop a bitcoind test node""" self.nodes[i].stop_node(expected_stderr, wait=wait) - self.nodes[i].wait_until_stopped() def stop_nodes(self, wait=0): """Stop multiple bitcoind test nodes""" for node in self.nodes: # Issue RPC to stop nodes - node.stop_node(wait=wait) + node.stop_node(wait=wait, wait_until_stopped=False) for node in self.nodes: # Wait for nodes to stop @@ -782,6 +781,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_sqlite_compiled(): raise SkipTest("sqlite has not been compiled.") + def skip_if_no_bdb(self): + """Skip the running test if BDB has not been compiled.""" + if not self.is_bdb_compiled(): + raise SkipTest("BDB has not been compiled.") + def skip_if_no_wallet_tool(self): """Skip the running test if bitcoin-wallet has not been compiled.""" if not self.is_wallet_tool_compiled(): @@ -822,5 +826,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): return self.config["components"].getboolean("ENABLE_ZMQ") def is_sqlite_compiled(self): - """Checks whether the wallet module was compiled.""" + """Checks whether the wallet module was compiled with Sqlite support.""" return self.config["components"].getboolean("USE_SQLITE") + + def is_bdb_compiled(self): + """Checks whether the wallet module was compiled with BDB support.""" + return self.config["components"].getboolean("USE_BDB") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 046efe730e..b61d433652 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -71,6 +71,7 @@ class TestNode(): """ self.index = i + self.p2p_conn_index = 1 self.datadir = datadir self.bitcoinconf = os.path.join(self.datadir, "bitcoin.conf") self.stdout_dir = os.path.join(self.datadir, "stdout") @@ -308,7 +309,7 @@ class TestNode(): def version_is_at_least(self, ver): return self.version is None or self.version >= ver - def stop_node(self, expected_stderr='', wait=0): + def stop_node(self, expected_stderr='', *, wait=0, wait_until_stopped=True): """Stop the node.""" if not self.running: return @@ -337,6 +338,9 @@ class TestNode(): del self.p2ps[:] + if wait_until_stopped: + self.wait_until_stopped() + def is_node_stopped(self): """Checks whether the node has stopped. @@ -482,11 +486,8 @@ class TestNode(): tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False) as log_stdout: try: self.start(extra_args, stdout=log_stdout, stderr=log_stderr, *args, **kwargs) - self.wait_for_rpc_connection() - self.stop_node() - self.wait_until_stopped() - except FailedToStartError as e: - self.log.debug('bitcoind failed to start: %s', e) + ret = self.process.wait(timeout=self.rpc_timeout) + self.log.debug(self._node_msg(f'bitcoind exited with status {ret} during initialization')) self.running = False self.process = None # Check stderr for expected message @@ -505,15 +506,19 @@ class TestNode(): if expected_msg != stderr: self._raise_assertion_error( 'Expected message "{}" does not fully match stderr:\n"{}"'.format(expected_msg, stderr)) - else: + except subprocess.TimeoutExpired: + self.process.kill() + self.running = False + self.process = None + assert_msg = f'bitcoind should have exited within {self.rpc_timeout}s ' if expected_msg is None: - assert_msg = "bitcoind should have exited with an error" + assert_msg += "with an error" else: - assert_msg = "bitcoind should have exited with expected error " + expected_msg + assert_msg += "with expected error " + expected_msg self._raise_assertion_error(assert_msg) def add_p2p_connection(self, p2p_conn, *, wait_for_verack=True, **kwargs): - """Add a p2p connection to the node. + """Add an inbound p2p connection to the node. This method adds the p2p connection to the self.p2ps list and also returns the connection to the caller.""" @@ -542,15 +547,39 @@ class TestNode(): return p2p_conn + def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): + """Add an outbound p2p connection from node. Either + full-relay("outbound-full-relay") or + block-relay-only("block-relay-only") connection. + + This method adds the p2p connection to the self.p2ps list and returns + the connection to the caller. + """ + + def addconnection_callback(address, port): + self.log.debug("Connecting to %s:%d %s" % (address, port, connection_type)) + self.addconnection('%s:%d' % (address, port), connection_type) + + p2p_conn.peer_accept_connection(connect_cb=addconnection_callback, connect_id=p2p_idx + 1, net=self.chain, timeout_factor=self.timeout_factor, **kwargs)() + + p2p_conn.wait_for_connect() + self.p2ps.append(p2p_conn) + + p2p_conn.wait_for_verack() + p2p_conn.sync_with_ping() + + return p2p_conn + def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" - return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION]) + return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION.decode("utf-8")]) def disconnect_p2ps(self): """Close all p2p connections to the node.""" for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] + wait_until_helper(lambda: self.num_test_p2p_connections() == 0, timeout_factor=self.timeout_factor) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 62ff5c6e33..b3eb2d61a7 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -362,6 +362,7 @@ def initialize_datadir(dirname, n, chain): f.write("listenonion=0\n") f.write("printtoconsole=0\n") f.write("upnp=0\n") + f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 39b3bf2a5b..7cb74bdcb3 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -40,9 +40,20 @@ class MiniWallet: self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) return blocks - def get_utxo(self): - """Return the last utxo. Can be used to get the change output immediately after a send_self_transfer""" - return self._utxos.pop() + def get_utxo(self, *, txid=''): + """ + Returns a utxo and marks it as spent (pops it from the internal list) + + Args: + txid (string), optional: get the first utxo we find from a specific transaction + + Note: Can be used to get the change output immediately after a send_self_transfer + """ + index = -1 # by default the last utxo + if txid: + utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos)) + index = self._utxos.index(utxo) + return self._utxos.pop(index) def send_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None): """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" @@ -60,9 +71,9 @@ class MiniWallet: tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] tx_hex = tx.serialize().hex() - txid = from_node.sendrawtransaction(tx_hex) - self._utxos.append({'txid': txid, 'vout': 0, 'value': send_value}) - tx_info = from_node.getmempoolentry(txid) + tx_info = from_node.testmempoolaccept([tx_hex])[0] + self._utxos.append({'txid': tx_info['txid'], 'vout': 0, 'value': send_value}) + from_node.sendrawtransaction(tx_hex) assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fee'], fee) - return {'txid': txid, 'wtxid': tx_info['wtxid'], 'hex': tx_hex} + assert_equal(tx_info['fees']['base'], fee) + return {'txid': tx_info['txid'], 'wtxid': from_node.decoderawtransaction(tx_hex)['hash'], 'hex': tx_hex} diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5b3db282e1..9bbf862568 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -261,6 +261,7 @@ BASE_SCRIPTS = [ 'feature_filelock.py', 'feature_loadblock.py', 'p2p_dos_header_tree.py', + 'p2p_add_connections.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', 'feature_includeconf.py', @@ -279,7 +280,6 @@ BASE_SCRIPTS = [ 'feature_config_args.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', - 'rpc_getpeerinfo_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 615b772dc8..8a1af24dcf 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -10,6 +10,8 @@ import stat import subprocess import textwrap +from collections import OrderedDict + from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -28,8 +30,11 @@ class ToolWalletTest(BitcoinTestFramework): def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] - args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] + list(args) - return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + default_args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] + if self.options.descriptors and 'create' in args: + default_args.append('-descriptors') + + return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) @@ -63,6 +68,119 @@ class ToolWalletTest(BitcoinTestFramework): result = 'unchanged' if new == old else 'increased!' self.log.debug('Wallet file timestamp {}'.format(result)) + def get_expected_info_output(self, name="", transactions=0, keypool=2, address=0): + wallet_name = self.default_wallet_name if name == "" else name + output_types = 3 # p2pkh, p2sh, segwit + if self.options.descriptors: + return textwrap.dedent('''\ + Wallet info + =========== + Name: %s + Format: sqlite + Descriptors: yes + Encrypted: no + HD (hd seed available): yes + Keypool Size: %d + Transactions: %d + Address Book: %d + ''' % (wallet_name, keypool * output_types, transactions, address)) + else: + return textwrap.dedent('''\ + Wallet info + =========== + Name: %s + Format: bdb + Descriptors: no + Encrypted: no + HD (hd seed available): yes + Keypool Size: %d + Transactions: %d + Address Book: %d + ''' % (wallet_name, keypool, transactions, address * output_types)) + + def read_dump(self, filename): + dump = OrderedDict() + with open(filename, "r", encoding="utf8") as f: + for row in f: + row = row.strip() + key, value = row.split(',') + dump[key] = value + return dump + + def assert_is_sqlite(self, filename): + with open(filename, 'rb') as f: + file_magic = f.read(16) + assert file_magic == b'SQLite format 3\x00' + + def assert_is_bdb(self, filename): + with open(filename, 'rb') as f: + f.seek(12, 0) + file_magic = f.read(4) + assert file_magic == b'\x00\x05\x31\x62' or file_magic == b'\x62\x31\x05\x00' + + def write_dump(self, dump, filename, magic=None, skip_checksum=False): + if magic is None: + magic = "BITCOIN_CORE_WALLET_DUMP" + with open(filename, "w", encoding="utf8") as f: + row = ",".join([magic, dump[magic]]) + "\n" + f.write(row) + for k, v in dump.items(): + if k == magic or k == "checksum": + continue + row = ",".join([k, v]) + "\n" + f.write(row) + if not skip_checksum: + row = ",".join(["checksum", dump["checksum"]]) + "\n" + f.write(row) + + def assert_dump(self, expected, received): + e = expected.copy() + r = received.copy() + + # BDB will add a "version" record that is not present in sqlite + # In that case, we should ignore this record in both + # But because this also effects the checksum, we also need to drop that. + v_key = "0776657273696f6e" # Version key + if v_key in e and v_key not in r: + del e[v_key] + del e["checksum"] + del r["checksum"] + if v_key not in e and v_key in r: + del r[v_key] + del e["checksum"] + del r["checksum"] + + assert_equal(len(e), len(r)) + for k, v in e.items(): + assert_equal(v, r[k]) + + def do_tool_createfromdump(self, wallet_name, dumpfile, file_format=None): + dumppath = os.path.join(self.nodes[0].datadir, dumpfile) + rt_dumppath = os.path.join(self.nodes[0].datadir, "rt-{}.dump".format(wallet_name)) + + dump_data = self.read_dump(dumppath) + + args = ["-wallet={}".format(wallet_name), + "-dumpfile={}".format(dumppath)] + if file_format is not None: + args.append("-format={}".format(file_format)) + args.append("createfromdump") + + load_output = "" + if file_format is not None and file_format != dump_data["format"]: + load_output += "Warning: Dumpfile wallet format \"{}\" does not match command line specified format \"{}\".\n".format(dump_data["format"], file_format) + self.assert_tool_output(load_output, *args) + assert os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", wallet_name)) + + self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", '-wallet={}'.format(wallet_name), '-dumpfile={}'.format(rt_dumppath), 'dump') + + rt_dump_data = self.read_dump(rt_dumppath) + wallet_dat = os.path.join(self.nodes[0].datadir, "regtest/wallets/", wallet_name, "wallet.dat") + if rt_dump_data["format"] == "bdb": + self.assert_is_bdb(wallet_dat) + else: + self.assert_is_sqlite(wallet_dat) + def test_invalid_tool_commands_and_args(self): self.log.info('Testing that various invalid commands raise with specific error messages') self.assert_raises_tool_error('Invalid command: foo', 'foo') @@ -98,33 +216,7 @@ class ToolWalletTest(BitcoinTestFramework): # shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling info: {}'.format(timestamp_before)) - if self.options.descriptors: - out = textwrap.dedent('''\ - Wallet info - =========== - Name: default_wallet - Format: sqlite - Descriptors: yes - Encrypted: no - HD (hd seed available): yes - Keypool Size: 6 - Transactions: 0 - Address Book: 1 - ''') - else: - out = textwrap.dedent('''\ - Wallet info - =========== - Name: \ - - Format: bdb - Descriptors: no - Encrypted: no - HD (hd seed available): yes - Keypool Size: 2 - Transactions: 0 - Address Book: 3 - ''') + out = self.get_expected_info_output(address=1) self.assert_tool_output(out, '-wallet=' + self.default_wallet_name, 'info') timestamp_after = self.wallet_timestamp() self.log.debug('Wallet file timestamp after calling info: {}'.format(timestamp_after)) @@ -155,33 +247,7 @@ class ToolWalletTest(BitcoinTestFramework): shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling info: {}'.format(timestamp_before)) - if self.options.descriptors: - out = textwrap.dedent('''\ - Wallet info - =========== - Name: default_wallet - Format: sqlite - Descriptors: yes - Encrypted: no - HD (hd seed available): yes - Keypool Size: 6 - Transactions: 1 - Address Book: 1 - ''') - else: - out = textwrap.dedent('''\ - Wallet info - =========== - Name: \ - - Format: bdb - Descriptors: no - Encrypted: no - HD (hd seed available): yes - Keypool Size: 2 - Transactions: 1 - Address Book: 3 - ''') + out = self.get_expected_info_output(transactions=1, address=1) self.assert_tool_output(out, '-wallet=' + self.default_wallet_name, 'info') shasum_after = self.wallet_shasum() timestamp_after = self.wallet_timestamp() @@ -199,19 +265,7 @@ class ToolWalletTest(BitcoinTestFramework): shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling create: {}'.format(timestamp_before)) - out = textwrap.dedent('''\ - Topping up keypool... - Wallet info - =========== - Name: foo - Format: bdb - Descriptors: no - Encrypted: no - HD (hd seed available): yes - Keypool Size: 2000 - Transactions: 0 - Address Book: 0 - ''') + out = "Topping up keypool...\n" + self.get_expected_info_output(name="foo", keypool=2000) self.assert_tool_output(out, '-wallet=foo', 'create') shasum_after = self.wallet_shasum() timestamp_after = self.wallet_timestamp() @@ -237,9 +291,13 @@ class ToolWalletTest(BitcoinTestFramework): self.log.debug('Wallet file timestamp after calling getwalletinfo: {}'.format(timestamp_after)) assert_equal(0, out['txcount']) - assert_equal(1000, out['keypoolsize']) - assert_equal(1000, out['keypoolsize_hd_internal']) - assert_equal(True, 'hdseedid' in out) + if not self.options.descriptors: + assert_equal(1000, out['keypoolsize']) + assert_equal(1000, out['keypoolsize_hd_internal']) + assert_equal(True, 'hdseedid' in out) + else: + assert_equal(3000, out['keypoolsize']) + assert_equal(3000, out['keypoolsize_hd_internal']) self.log_wallet_timestamp_comparison(timestamp_before, timestamp_after) assert_equal(timestamp_before, timestamp_after) @@ -255,18 +313,95 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_tool_output('', '-wallet=salvage', 'salvage') + def test_dump_createfromdump(self): + self.start_node(0) + self.nodes[0].createwallet("todump") + file_format = self.nodes[0].get_wallet_rpc("todump").getwalletinfo()["format"] + self.nodes[0].createwallet("todump2") + self.stop_node(0) + + self.log.info('Checking dump arguments') + self.assert_raises_tool_error('No dump file provided. To use dump, -dumpfile=<filename> must be provided.', '-wallet=todump', 'dump') + + self.log.info('Checking basic dump') + wallet_dump = os.path.join(self.nodes[0].datadir, "wallet.dump") + self.assert_tool_output('The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n', '-wallet=todump', '-dumpfile={}'.format(wallet_dump), 'dump') + + dump_data = self.read_dump(wallet_dump) + orig_dump = dump_data.copy() + # Check the dump magic + assert_equal(dump_data['BITCOIN_CORE_WALLET_DUMP'], '1') + # Check the file format + assert_equal(dump_data["format"], file_format) + + self.log.info('Checking that a dumpfile cannot be overwritten') + self.assert_raises_tool_error('File {} already exists. If you are sure this is what you want, move it out of the way first.'.format(wallet_dump), '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'dump') + + self.log.info('Checking createfromdump arguments') + self.assert_raises_tool_error('No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided.', '-wallet=todump', 'createfromdump') + non_exist_dump = os.path.join(self.nodes[0].datadir, "wallet.nodump") + self.assert_raises_tool_error('Unknown wallet file format "notaformat" provided. Please provide one of "bdb" or "sqlite".', '-wallet=todump', '-format=notaformat', '-dumpfile={}'.format(wallet_dump), 'createfromdump') + self.assert_raises_tool_error('Dump file {} does not exist.'.format(non_exist_dump), '-wallet=todump', '-dumpfile={}'.format(non_exist_dump), 'createfromdump') + wallet_path = os.path.join(self.nodes[0].datadir, 'regtest/wallets/todump2') + self.assert_raises_tool_error('Failed to create database path \'{}\'. Database already exists.'.format(wallet_path), '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') + self.assert_raises_tool_error("The -descriptors option can only be used with the 'create' command.", '-descriptors', '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') + + self.log.info('Checking createfromdump') + self.do_tool_createfromdump("load", "wallet.dump") + self.do_tool_createfromdump("load-bdb", "wallet.dump", "bdb") + if self.is_sqlite_compiled(): + self.do_tool_createfromdump("load-sqlite", "wallet.dump", "sqlite") + + self.log.info('Checking createfromdump handling of magic and versions') + bad_ver_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_ver1.dump") + dump_data["BITCOIN_CORE_WALLET_DUMP"] = "0" + self.write_dump(dump_data, bad_ver_wallet_dump) + self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 0', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + bad_ver_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_ver2.dump") + dump_data["BITCOIN_CORE_WALLET_DUMP"] = "2" + self.write_dump(dump_data, bad_ver_wallet_dump) + self.assert_raises_tool_error('Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version 2', '-wallet=badload', '-dumpfile={}'.format(bad_ver_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + bad_magic_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_magic.dump") + del dump_data["BITCOIN_CORE_WALLET_DUMP"] + dump_data["not_the_right_magic"] = "1" + self.write_dump(dump_data, bad_magic_wallet_dump, "not_the_right_magic") + self.assert_raises_tool_error('Error: Dumpfile identifier record is incorrect. Got "not_the_right_magic", expected "BITCOIN_CORE_WALLET_DUMP".', '-wallet=badload', '-dumpfile={}'.format(bad_magic_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + + self.log.info('Checking createfromdump handling of checksums') + bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum1.dump") + dump_data = orig_dump.copy() + checksum = dump_data["checksum"] + dump_data["checksum"] = "1" * 64 + self.write_dump(dump_data, bad_sum_wallet_dump) + self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}'.format(checksum, "1" * 64), '-wallet=bad', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum2.dump") + del dump_data["checksum"] + self.write_dump(dump_data, bad_sum_wallet_dump, skip_checksum=True) + self.assert_raises_tool_error('Error: Missing checksum', '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + bad_sum_wallet_dump = os.path.join(self.nodes[0].datadir, "wallet-bad_sum3.dump") + dump_data["checksum"] = "2" * 10 + self.write_dump(dump_data, bad_sum_wallet_dump) + self.assert_raises_tool_error('Error: Dumpfile checksum does not match. Computed {}, expected {}{}'.format(checksum, "2" * 10, "0" * 54), '-wallet=badload', '-dumpfile={}'.format(bad_sum_wallet_dump), 'createfromdump') + assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "regtest/wallets", "badload")) + + def run_test(self): self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename) self.test_invalid_tool_commands_and_args() # Warning: The following tests are order-dependent. self.test_tool_wallet_info() self.test_tool_wallet_info_after_transaction() + self.test_tool_wallet_create_on_existing_wallet() + self.test_getwalletinfo_on_different_wallet() if not self.options.descriptors: - # TODO: Wallet tool needs more create options at which point these can be enabled. - self.test_tool_wallet_create_on_existing_wallet() - self.test_getwalletinfo_on_different_wallet() # Salvage is a legacy wallet only thing self.test_salvage() + self.test_dump_createfromdump() if __name__ == '__main__': ToolWalletTest().main() diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 0e786ca595..433b40faee 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -263,8 +263,6 @@ class WalletTest(BitcoinTestFramework): self.log.info('Put txs back into mempool of node 1 (not node 0)') self.nodes[0].invalidateblock(block_reorg) self.nodes[1].invalidateblock(block_reorg) - self.sync_blocks() - self.nodes[0].syncwithvalidationinterfacequeue() assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted self.nodes[0].generatetoaddress(1, ADDRESS_WATCHONLY) assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index ead56046a4..6bcb03e8ed 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" @@ -235,8 +235,8 @@ class WalletTest(BitcoinTestFramework): fee_rate_btc_kvb = fee_rate_sat_vb * 1e3 / 1e8 explicit_fee_rate_btc_kvb = Decimal(fee_rate_btc_kvb) / 1000 - # Passing conf_target 0, estimate_mode "" as placeholder arguments should allow fee_rate to apply. - txid = self.nodes[2].sendmany(amounts={address: 10}, conf_target=0, estimate_mode="", fee_rate=fee_rate_sat_vb) + # Test passing fee_rate as a string + txid = self.nodes[2].sendmany(amounts={address: 10}, fee_rate=str(fee_rate_sat_vb)) self.nodes[2].generate(1) self.sync_all(self.nodes[0:3]) balance = self.nodes[2].getbalance() @@ -245,6 +245,17 @@ class WalletTest(BitcoinTestFramework): node_0_bal += Decimal('10') assert_equal(self.nodes[0].getbalance(), node_0_bal) + # Test passing fee_rate as an integer + amount = Decimal("0.0001") + txid = self.nodes[2].sendmany(amounts={address: amount}, fee_rate=fee_rate_sat_vb) + self.nodes[2].generate(1) + self.sync_all(self.nodes[0:3]) + balance = self.nodes[2].getbalance() + node_2_bal = self.check_fee_amount(balance, node_2_bal - amount, explicit_fee_rate_btc_kvb, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) + assert_equal(balance, node_2_bal) + node_0_bal += amount + assert_equal(self.nodes[0].getbalance(), node_0_bal) + for key in ["totalFee", "feeRate"]: assert_raises_rpc_error(-8, "Unknown named parameter key", self.nodes[2].sendtoaddress, address=address, amount=1, fee_rate=1, key=1) @@ -406,9 +417,8 @@ class WalletTest(BitcoinTestFramework): amount = 3 fee_rate_sat_vb = 2 fee_rate_btc_kvb = fee_rate_sat_vb * 1e3 / 1e8 - - # Passing conf_target 0, estimate_mode "" as placeholder arguments should allow fee_rate to apply. - txid = self.nodes[2].sendtoaddress(address=address, amount=amount, conf_target=0, estimate_mode="", fee_rate=fee_rate_sat_vb) + # Test passing fee_rate as an integer + txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=fee_rate_sat_vb) tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) self.nodes[0].generate(1) self.sync_all(self.nodes[0:3]) @@ -416,6 +426,19 @@ class WalletTest(BitcoinTestFramework): fee = prebalance - postbalance - Decimal(amount) assert_fee_amount(fee, tx_size, Decimal(fee_rate_btc_kvb)) + prebalance = self.nodes[2].getbalance() + amount = Decimal("0.001") + fee_rate_sat_vb = 1.23 + fee_rate_btc_kvb = fee_rate_sat_vb * 1e3 / 1e8 + # Test passing fee_rate as a string + txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=str(fee_rate_sat_vb)) + tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) + postbalance = self.nodes[2].getbalance() + fee = prebalance - postbalance - amount + assert_fee_amount(fee, tx_size, Decimal(fee_rate_btc_kvb)) + for key in ["totalFee", "feeRate"]: assert_raises_rpc_error(-8, "Unknown named parameter key", self.nodes[2].sendtoaddress, address=address, amount=1, fee_rate=1, key=1) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c323aac3cc..c8c1f2e374 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -107,12 +107,10 @@ class BumpFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Unexpected key {}".format(key), rbf_node.bumpfee, rbfid, {key: NORMAL}) # Bumping to just above minrelay should fail to increase the total fee enough. - assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141, must be at least 0.00001704 (oldFee 0.00000999 + incrementalFee 0.00000705)", - rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) + assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) self.log.info("Test invalid fee rate settings") - assert_raises_rpc_error(-8, "Insufficient total fee 0.00, must be at least 0.00001704 (oldFee 0.00000999 + incrementalFee 0.00000705)", - rbf_node.bumpfee, rbfid, {"fee_rate": 0}) + assert_raises_rpc_error(-8, "Insufficient total fee 0.00", rbf_node.bumpfee, rbfid, {"fee_rate": 0}) assert_raises_rpc_error(-4, "Specified or calculated fee 0.141 is too high (cannot be higher than -maxtxfee 0.10", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) @@ -151,7 +149,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() if mode == "fee_rate": - bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": NORMAL}) + bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": str(NORMAL)}) bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": NORMAL}) else: bumped_psbt = rbf_node.psbtbumpfee(rbfid) @@ -421,7 +419,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.sync_all() # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"fee_rate": 1}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 0f11aca525..a39a3c8d9b 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 0fec8ea4a4..1de41a5f96 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descriptor wallet function.""" diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index c2b30fb35b..78cf378642 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index 4509c1e0b2..0d702e44f6 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Wallet encryption""" @@ -78,7 +78,7 @@ class WalletEncryptionTest(BitcoinTestFramework): MAX_VALUE = 100000000 expected_time = int(time.time()) + MAX_VALUE - 600 self.nodes[0].walletpassphrase(passphrase2, MAX_VALUE - 600) - # give buffer for walletpassphrase, since it iterates over all crypted keys + # give buffer for walletpassphrase, since it iterates over all encrypted keys expected_time_with_buffer = time.time() + MAX_VALUE - 600 actual_time = self.nodes[0].getwalletinfo()['unlocked_until'] assert_greater_than_or_equal(actual_time, expected_time) diff --git a/test/functional/wallet_fallbackfee.py b/test/functional/wallet_fallbackfee.py index dbf853b35c..78eef4b790 100755 --- a/test/functional/wallet_fallbackfee.py +++ b/test/functional/wallet_fallbackfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet replace-by-fee capabilities in conjunction with the fallbackfee.""" diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index aad112b499..59089456e9 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet import RPCs. diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 2903a84998..ed62ef0e9d 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importdescriptors RPC. diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f7fdd6e908..13186b9e1d 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC. @@ -64,7 +64,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[0].generate(1) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() + self.nodes[1].syncwithvalidationinterfacequeue() # Sync the timestamp to the wallet, so that importmulti works node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index d10d8ae1c8..7635ce2139 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importprunedfunds and removeprunedfunds RPCs.""" diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 51795aca23..3fe6adeebc 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 78e06c5916..5619d57947 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 6a1b9097c5..448720530c 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listsinceblock RPC.""" diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 3e98208431..71573964de 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index df16ec741f..bb89e76a9a 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -23,9 +23,11 @@ from test_framework.util import ( ) got_loading_error = False + + def test_load_unload(node, name): global got_loading_error - for _ in range(10): + while True: if got_loading_error: return try: @@ -68,7 +70,7 @@ class MultiWalletTest(BitcoinTestFramework): return wallet_dir(name, "wallet.dat") return wallet_dir(name) - assert_equal(self.nodes[0].listwalletdir(), { 'wallets': [{ 'name': self.default_wallet_name }] }) + assert_equal(self.nodes[0].listwalletdir(), {'wallets': [{'name': self.default_wallet_name}]}) # check wallet.dat is created self.stop_nodes() @@ -127,7 +129,7 @@ class MultiWalletTest(BitcoinTestFramework): os.mkdir(wallet_dir('no_access')) os.chmod(wallet_dir('no_access'), 0) try: - with self.nodes[0].assert_debug_log(expected_msgs=['Too many levels of symbolic links', 'Error scanning']): + with self.nodes[0].assert_debug_log(expected_msgs=['Error scanning']): walletlist = self.nodes[0].listwalletdir()['wallets'] finally: # Need to ensure access is restored for cleanup @@ -278,7 +280,7 @@ class MultiWalletTest(BitcoinTestFramework): threads = [] for _ in range(3): n = node.cli if self.options.usecli else get_rpc_proxy(node.url, 1, timeout=600, coveragedir=node.coverage_dir) - t = Thread(target=test_load_unload, args=(n, wallet_names[2], )) + t = Thread(target=test_load_unload, args=(n, wallet_names[2])) t.start() threads.append(t) for t in threads: @@ -355,12 +357,18 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet) assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy") assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet) - assert_raises_rpc_error(-8, "Cannot unload the requested wallet", w1.unloadwallet, "w2"), + assert_raises_rpc_error(-8, "RPC endpoint wallet and wallet_name parameter specify different wallets", w1.unloadwallet, "w2"), # Successfully unload the specified wallet name self.nodes[0].unloadwallet("w1") assert 'w1' not in self.nodes[0].listwallets() + # Unload w1 again, this time providing the wallet name twice + self.nodes[0].loadwallet("w1") + assert 'w1' in self.nodes[0].listwallets() + w1.unloadwallet("w1") + assert 'w1' not in self.nodes[0].listwallets() + # Successfully unload the wallet referenced by the request endpoint # Also ensure unload works during walletpassphrase timeout w2.encryptwallet('test') diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index a1d6b098ad..9a5866a361 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 353deefcc1..78d88f8aa5 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -50,6 +50,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): block.solve() node.submitblock(ToHex(block)) + # Set correct m_best_block_time, which is used in ResendWalletTransactions node.syncwithvalidationinterfacequeue() now = int(time.time()) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 5da71c85f9..9835c5a2af 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -256,21 +256,20 @@ class WalletSendTest(BitcoinTestFramework): assert res["complete"] self.log.info("Test setting explicit fee rate") - res1 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=1, add_to_wallet=False) - res2 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=1, add_to_wallet=False) + res1 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate="1", add_to_wallet=False) + res2 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate="1", add_to_wallet=False) assert_equal(self.nodes[1].decodepsbt(res1["psbt"])["fee"], self.nodes[1].decodepsbt(res2["psbt"])["fee"]) - # Passing conf_target 0, estimate_mode "" as placeholder arguments should allow fee_rate to apply. - res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0, estimate_mode="", fee_rate=7, add_to_wallet=False) + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=7, add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00007")) - res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=2, add_to_wallet=False) + # "unset" and None are treated the same for estimate_mode + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=2, estimate_mode="unset", add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00002")) - # Passing conf_target 0, estimate_mode "" as placeholder arguments should allow fee_rate to apply. - res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_conf_target=0, arg_estimate_mode="", arg_fee_rate=4.531, add_to_wallet=False) + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=4.531, add_to_wallet=False) fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00004531")) @@ -303,10 +302,16 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=v, estimate_mode=mode, expect_error=(-3, "Expected type number for conf_target, got {}".format(k))) - # Test setting explicit fee rate just below the minimum. + # Test setting explicit fee rate just below the minimum and at zero. self.log.info("Explicit fee rate raises RPC error 'fee rate too low' if fee_rate of 0.99999999 is passed") self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=0.99999999, expect_error=(-4, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=0.99999999, + expect_error=(-4, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=0, + expect_error=(-4, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=0, + expect_error=(-4, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) # TODO: Return hex if fee rate is below -maxmempool # res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0.1, estimate_mode="sat/b", add_to_wallet=False) diff --git a/test/functional/wallet_startup.py b/test/functional/wallet_startup.py index d3119925f7..d96c2da686 100755 --- a/test/functional/wallet_startup.py +++ b/test/functional/wallet_startup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet load on startup. diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index bdbbb3e530..893a2d9617 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index 42de131354..c7f7a8546a 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 8ab4b3f76c..d0bb6135a8 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -22,9 +22,7 @@ from test_framework.messages import deser_compact_size, deser_string from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - assert_greater_than, assert_is_hex_string, - assert_raises_rpc_error, sha256sum_file, ) @@ -92,6 +90,32 @@ class UpgradeWalletTest(BitcoinTestFramework): v16_3_node.submitblock(b) assert_equal(v16_3_node.getblockcount(), to_height) + def test_upgradewallet(self, wallet, previous_version, requested_version=None, expected_version=None): + unchanged = expected_version == previous_version + new_version = previous_version if unchanged else expected_version if expected_version else requested_version + assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) + assert_equal(wallet.upgradewallet(requested_version), + { + "wallet_name": "", + "previous_version": previous_version, + "current_version": new_version, + "result": "Already at latest version. Wallet version unchanged." if unchanged else "Wallet upgraded successfully from version {} to version {}.".format(previous_version, new_version), + } + ) + assert_equal(wallet.getwalletinfo()["walletversion"], new_version) + + def test_upgradewallet_error(self, wallet, previous_version, requested_version, msg): + assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) + assert_equal(wallet.upgradewallet(requested_version), + { + "wallet_name": "", + "previous_version": previous_version, + "current_version": previous_version, + "error": msg, + } + ) + assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) + def run_test(self): self.nodes[0].generatetoaddress(101, self.nodes[0].getnewaddress()) self.dumb_sync_blocks() @@ -158,14 +182,8 @@ class UpgradeWalletTest(BitcoinTestFramework): self.restart_node(0) copy_v16() wallet = node_master.get_wallet_rpc(self.default_wallet_name) - old_version = wallet.getwalletinfo()["walletversion"] - - # calling upgradewallet without version arguments - # should return nothing if successful - assert_equal(wallet.upgradewallet(), {}) - new_version = wallet.getwalletinfo()["walletversion"] - # upgraded wallet version should be greater than older one - assert_greater_than(new_version, old_version) + self.log.info("Test upgradewallet without a version argument") + self.test_upgradewallet(wallet, previous_version=159900, expected_version=169900) # wallet should still contain the same balance assert_equal(wallet.getbalance(), v16_3_balance) @@ -173,32 +191,28 @@ class UpgradeWalletTest(BitcoinTestFramework): wallet = node_master.get_wallet_rpc(self.default_wallet_name) # should have no master key hash before conversion assert_equal('hdseedid' in wallet.getwalletinfo(), False) - # calling upgradewallet with explicit version number - # should return nothing if successful - assert_equal(wallet.upgradewallet(169900), {}) - new_version = wallet.getwalletinfo()["walletversion"] - # upgraded wallet should have version 169900 - assert_equal(new_version, 169900) + self.log.info("Test upgradewallet with explicit version number") + self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900) # after conversion master key hash should be present assert_is_hex_string(wallet.getwalletinfo()['hdseedid']) - self.log.info('Intermediary versions don\'t effect anything') + self.log.info("Intermediary versions don't effect anything") copy_non_hd() # Wallet starts with 60000 assert_equal(60000, wallet.getwalletinfo()['walletversion']) wallet.unloadwallet() before_checksum = sha256sum_file(node_master_wallet) node_master.loadwallet('') - # Can "upgrade" to 129999 which should have no effect on the wallet - wallet.upgradewallet(129999) - assert_equal(60000, wallet.getwalletinfo()['walletversion']) + # Test an "upgrade" from 60000 to 129999 has no effect, as the next version is 130000 + self.test_upgradewallet(wallet, previous_version=60000, requested_version=129999, expected_version=60000) wallet.unloadwallet() assert_equal(before_checksum, sha256sum_file(node_master_wallet)) node_master.loadwallet('') self.log.info('Wallets cannot be downgraded') copy_non_hd() - assert_raises_rpc_error(-4, 'Cannot downgrade wallet', wallet.upgradewallet, 40000) + self.test_upgradewallet_error(wallet, previous_version=60000, requested_version=40000, + msg="Cannot downgrade wallet from version 60000 to version 40000. Wallet version unchanged.") wallet.unloadwallet() assert_equal(before_checksum, sha256sum_file(node_master_wallet)) node_master.loadwallet('') @@ -208,8 +222,7 @@ class UpgradeWalletTest(BitcoinTestFramework): orig_kvs = dump_bdb_kv(node_master_wallet) assert b'\x07hdchain' not in orig_kvs # Upgrade to HD, no split - wallet.upgradewallet(130000) - assert_equal(130000, wallet.getwalletinfo()['walletversion']) + self.test_upgradewallet(wallet, previous_version=60000, requested_version=130000) # Check that there is now a hd chain and it is version 1, no internal chain counter new_kvs = dump_bdb_kv(node_master_wallet) assert b'\x07hdchain' in new_kvs @@ -236,16 +249,13 @@ class UpgradeWalletTest(BitcoinTestFramework): assert_equal('m/0\'/0\'/1\'', info['hdkeypath']) self.log.info('Cannot upgrade to HD Split, needs Pre Split Keypool') - assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 139900) - assert_equal(130000, wallet.getwalletinfo()['walletversion']) - assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 159900) - assert_equal(130000, wallet.getwalletinfo()['walletversion']) - assert_raises_rpc_error(-4, 'Cannot upgrade a non HD split wallet without upgrading to support pre split keypool', wallet.upgradewallet, 169899) - assert_equal(130000, wallet.getwalletinfo()['walletversion']) + for version in [139900, 159900, 169899]: + self.test_upgradewallet_error(wallet, previous_version=130000, requested_version=version, + msg="Cannot upgrade a non HD split wallet from version {} to version {} without upgrading to " + "support pre-split keypool. Please use version 169900 or no version specified.".format(130000, version)) self.log.info('Upgrade HD to HD chain split') - wallet.upgradewallet(169900) - assert_equal(169900, wallet.getwalletinfo()['walletversion']) + self.test_upgradewallet(wallet, previous_version=130000, requested_version=169900) # Check that the hdchain updated correctly new_kvs = dump_bdb_kv(node_master_wallet) hd_chain = new_kvs[b'\x07hdchain'] @@ -271,8 +281,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.log.info('Upgrade non-HD to HD chain split') copy_non_hd() - wallet.upgradewallet(169900) - assert_equal(169900, wallet.getwalletinfo()['walletversion']) + self.test_upgradewallet(wallet, previous_version=60000, requested_version=169900) # Check that the hdchain updated correctly new_kvs = dump_bdb_kv(node_master_wallet) hd_chain = new_kvs[b'\x07hdchain'] @@ -333,8 +342,8 @@ class UpgradeWalletTest(BitcoinTestFramework): # Check the wallet has a default key initially old_kvs = dump_bdb_kv(node_master_wallet) defaultkey = old_kvs[b'\x0adefaultkey'] - # Upgrade the wallet. Should still have the same default key - wallet.upgradewallet(159900) + self.log.info("Upgrade the wallet. Should still have the same default key.") + self.test_upgradewallet(wallet, previous_version=139900, requested_version=159900) new_kvs = dump_bdb_kv(node_master_wallet) up_defaultkey = new_kvs[b'\x0adefaultkey'] assert_equal(defaultkey, up_defaultkey) @@ -342,5 +351,6 @@ class UpgradeWalletTest(BitcoinTestFramework): v16_3_kvs = dump_bdb_kv(v16_3_wallet) assert b'\x0adefaultkey' not in v16_3_kvs + if __name__ == '__main__': UpgradeWalletTest().main() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index c7895edbcc..3c743603bb 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -54,7 +54,7 @@ def main(): ) parser.add_argument( '--m_dir', - help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', + help='Merge inputs from this directory into the seed_dir.', ) parser.add_argument( '-g', @@ -83,7 +83,7 @@ def main(): sys.exit(1) # Build list of tests - test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include')) + test_list_all = parse_test_list(fuzz_bin=os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz')) if not test_list_all: logging.error("No fuzz targets found") @@ -126,9 +126,12 @@ def main(): try: help_output = subprocess.run( args=[ - os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), + os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), '-help=1', ], + env={ + 'FUZZ': test_list_selection[0] + }, timeout=20, check=True, stderr=subprocess.PIPE, @@ -177,35 +180,41 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): """ logging.info("Generating corpus seeds to {}".format(seed_dir)) - def job(command): + def job(command, t): logging.debug("Running '{}'\n".format(" ".join(command))) logging.debug("Command '{}' output:\n'{}'\n".format( ' '.join(command), - subprocess.run(command, check=True, stderr=subprocess.PIPE, - universal_newlines=True).stderr - )) + subprocess.run( + command, + env={ + 'FUZZ': t + }, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stderr)) futures = [] for target in targets: target_seed_dir = os.path.join(seed_dir, target) os.makedirs(target_seed_dir, exist_ok=True) command = [ - os.path.join(build_dir, "src", "test", "fuzz", target), + os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), "-runs=100000", target_seed_dir, ] - futures.append(fuzz_pool.submit(job, command)) + futures.append(fuzz_pool.submit(job, command, target)) for future in as_completed(futures): future.result() def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): - logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) + logging.info("Merge the inputs from the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] for t in test_list: args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', t), + os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), '-merge=1', '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), @@ -216,7 +225,15 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): def job(t, args): output = 'Run {} with args {}\n'.format(t, " ".join(args)) - output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + output += subprocess.run( + args, + env={ + 'FUZZ': t + }, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stderr logging.debug(output) jobs.append(fuzz_pool.submit(job, t, args)) @@ -231,7 +248,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) args = [ - os.path.join(build_dir, 'src', 'test', 'fuzz', t), + os.path.join(build_dir, 'src', 'test', 'fuzz', 'fuzz'), '-runs=1', corpus_path, ] @@ -240,7 +257,7 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): def job(t, args): output = 'Run {} with args {}'.format(t, args) - result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) + result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True) output += result.stderr return output, result @@ -260,20 +277,16 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): sys.exit(1) -def parse_test_list(makefile): - with open(makefile, encoding='utf-8') as makefile_test: - test_list_all = [] - read_targets = False - for line in makefile_test.readlines(): - line = line.strip().replace('test/fuzz/', '').replace(' \\', '') - if read_targets: - if not line: - break - test_list_all.append(line) - continue - - if line == 'FUZZ_TARGETS =': - read_targets = True +def parse_test_list(*, fuzz_bin): + test_list_all = subprocess.run( + fuzz_bin, + env={ + 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '' + }, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + universal_newlines=True, + ).stdout.splitlines() return test_list_all diff --git a/test/lint/README.md b/test/lint/README.md index d15c061288..7e06308347 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -15,7 +15,16 @@ git-subtree-check.sh Run this script from the root of the repository to verify that a subtree matches the contents of the commit it claims to have been updated to. -To use, make sure that you have fetched the upstream repository branch in which the subtree is +``` +Usage: test/lint/git-subtree-check.sh [-r] DIR [COMMIT] + test/lint/git-subtree-check.sh -? +``` + +- `DIR` is the prefix within the repository to check. +- `COMMIT` is the commit to check, if it is not provided, HEAD will be used. +- `-r` checks that subtree commit is present in repository. + +To do a full check with `-r`, make sure that you have fetched the upstream repository branch in which the subtree is maintained: * for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) * for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) @@ -29,10 +38,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -Usage: `git-subtree-check.sh DIR (COMMIT)` - -`COMMIT` may be omitted, in which case `HEAD` is used. - lint-all.sh =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index f77242d335..feaebc68e9 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 827c978bed..e66adfc3ce 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh index 20021d8605..b2ed811cda 100755 --- a/test/lint/extended-lint-cppcheck.sh +++ b/test/lint/extended-lint-cppcheck.sh @@ -30,6 +30,7 @@ IGNORED_WARNINGS=( "src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit." "src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit." "src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit." + "src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit." "src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit." "src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit." "src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit." @@ -42,6 +43,11 @@ IGNORED_WARNINGS=( "src/script/descriptor.cpp:.* Class 'WSHDescriptor' has a constructor with 1 argument that is not explicit." "src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit." "src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit." + "src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit." + "src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit." + "src/span.h:.* Class 'Span < const std :: vector <unsigned char > >' has a constructor with 1 argument that is not explicit." + "src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit." + "src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit." "src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit." "src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit." "src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit." @@ -49,6 +55,9 @@ IGNORED_WARNINGS=( "src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit." "src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit." "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." + "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." + "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." + "src/util/ref.h:.* Class 'Ref' has a constructor with 1 argument that is not explicit." "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." ) @@ -66,7 +75,7 @@ function join_array { ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" | \ - xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++11 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ + xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ grep -E "${ENABLED_CHECKS_REGEXP}" | \ grep -vE "${IGNORED_WARNINGS_REGEXP}") if [[ ${WARNINGS} != "" ]]; then diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 5a0500df25..3556e49e08 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -1,9 +1,42 @@ #!/bin/sh -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C + +check_remote=0 +while getopts "?hr" opt; do + case $opt in + '?' | h) + echo "Usage: $0 [-r] DIR [COMMIT]" + echo " $0 -?" + echo "" + echo "Checks that a certain prefix is pure subtree, and optionally whether the" + echo "referenced commit is present in any fetched remote." + echo "" + echo "DIR is the prefix within the repository to check." + echo "COMMIT is the commit to check, if it is not provided, HEAD will be used." + echo "" + echo "-r Check that subtree commit is present in repository." + echo " To do this check, fetch the subtreed remote first. Example:" + echo "" + echo " git fetch https://github.com/bitcoin-core/secp256k1.git" + echo " test/lint/git-subtree-check.sh -r src/secp256k1" + exit 1 + ;; + r) + check_remote=1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "$1" ]; then + echo "Need to provide a DIR, see $0 -?" + exit 1 +fi + # Strip trailing / from directory path (in case it was added by autocomplete) DIR="${1%/}" COMMIT="$2" @@ -79,18 +112,20 @@ if [ "$tree_actual_tree" != "$tree_commit" ]; then exit 1 fi -# get the tree in the subtree commit referred to -if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2 - exit -fi -tree_subtree=$(git show -s --format="%T" $rev) -echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" +if [ "$check_remote" != "0" ]; then + # get the tree in the subtree commit referred to + if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then + echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2 + exit 1 + fi + tree_subtree=$(git show -s --format="%T" $rev) + echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" -# ... and compare the actual tree with it -if [ "$tree_actual_tree" != "$tree_subtree" ]; then - echo "FAIL: subtree update commit differs from upstream tree!" >&2 - exit 1 + # ... and compare the actual tree with it + if [ "$tree_actual_tree" != "$tree_subtree" ]; then + echo "FAIL: subtree update commit differs from upstream tree!" >&2 + exit 1 + fi fi echo "GOOD" diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index d30a8ca231..2860f5621b 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 6bd02d45ac..c4ad00e954 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -20,7 +20,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "txmempool -> validation -> txmempool" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "policy/fees -> txmempool -> validation -> policy/fees" ) EXIT_CODE=0 diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh index ecaad215c4..2b3a9b87c2 100755 --- a/test/lint/lint-git-commit-check.sh +++ b/test/lint/lint-git-commit-check.sh @@ -23,13 +23,6 @@ while getopts "?" opt; do esac done -# TRAVIS_BRANCH will be present in a Travis environment. For builds triggered -# by a pull request this is the name of the branch targeted by the pull request. -# https://docs.travis-ci.com/user/environment-variables/ -if [ -n "${TRAVIS_BRANCH}" ]; then - COMMIT_RANGE="$TRAVIS_BRANCH..HEAD" -fi - if [ -z "${COMMIT_RANGE}" ]; then if [ -n "$1" ]; then COMMIT_RANGE="HEAD~$1...HEAD" diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 5d5a150db8..5cfa41537f 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index fde77aea2d..dc032665e4 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -60,7 +60,6 @@ EXPECTED_BOOST_INCLUDES=( boost/multi_index/ordered_index.hpp boost/multi_index/sequenced_index.hpp boost/multi_index_container.hpp - boost/optional.hpp boost/preprocessor/cat.hpp boost/preprocessor/stringize.hpp boost/process.hpp @@ -72,9 +71,6 @@ EXPECTED_BOOST_INCLUDES=( boost/thread/mutex.hpp boost/thread/shared_mutex.hpp boost/thread/thread.hpp - boost/variant.hpp - boost/variant/apply_visitor.hpp - boost/variant/static_visitor.hpp ) for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh index 7257919c98..6e5b18fc23 100755 --- a/test/lint/lint-python-utf8-encoding.sh +++ b/test/lint/lint-python-utf8-encoding.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 4fc130497b..51815963f6 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -102,7 +102,7 @@ if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; e EXIT_CODE=1 fi -if ! mypy --ignore-missing-imports $(git ls-files "test/functional/*.py"); then +if ! mypy --ignore-missing-imports $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then EXIT_CODE=1 fi diff --git a/test/lint/lint-rpc-help.sh b/test/lint/lint-rpc-help.sh deleted file mode 100755 index faac5d43e2..0000000000 --- a/test/lint/lint-rpc-help.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check that all RPC help texts are generated by RPCHelpMan. - -export LC_ALL=C - -EXIT_CODE=0 - -# Assume that all multiline strings passed into a runtime_error are help texts. -# This is potentially fragile, but the linter is only temporary and can safely -# be removed early 2019. - -non_autogenerated_help=$(grep --perl-regexp --null-data --only-matching 'runtime_error\(\n\s*".*\\n"\n' $(git ls-files -- "*.cpp")) -if [[ ${non_autogenerated_help} != "" ]]; then - echo "Must use RPCHelpMan to generate the help for the following RPC methods:" - echo "${non_autogenerated_help}" - echo - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh index a5c8aa42b2..13ebdfd78a 100755 --- a/test/lint/lint-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 9a26cd9c02..4dbf5ed28e 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -8,14 +8,6 @@ export LC_ALL=C -# The shellcheck binary segfault/coredumps in Travis with LC_ALL=C -# It does not do so in Ubuntu 14.04, 16.04, 18.04 in versions 0.3.3, 0.3.7, 0.4.6 -# respectively. So export LC_ALL=C is set as required by lint-shell-locale.sh -# but unset here in case of running in Travis. -if [ "$TRAVIS" = "true" ]; then - unset LC_ALL -fi - # Disabled warnings: disabled=( SC2046 # Quote this to prevent word splitting. @@ -36,7 +28,8 @@ fi SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then +SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files +if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then EXIT_CODE=1 fi diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/lint-spelling.ignore-words.txt index 34f54325b3..78ffe4def3 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/lint-spelling.ignore-words.txt @@ -1,17 +1,15 @@ +asend +blockin +cachable +fo +fpr hights +hist +inout mor -mut -objext +nin +ser +unparseable +unser useable wit -unparseable -copyable -cachable -errorstring -keyserver -homogenous -setban -hist -ser -unselect -lowercased diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index cb84727ba5..fbdf3c59c1 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/gitian-keys/keys.txt"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index 80af0a439d..c800fd20db 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -22,13 +22,6 @@ while getopts "?" opt; do esac done -# TRAVIS_BRANCH will be present in a Travis environment. For builds triggered -# by a pull request this is the name of the branch targeted by the pull request. -# https://docs.travis-ci.com/user/environment-variables/ -if [ -n "${TRAVIS_BRANCH}" ]; then - COMMIT_RANGE="$TRAVIS_BRANCH..HEAD" -fi - if [ -z "${COMMIT_RANGE}" ]; then if [ -n "$1" ]; then COMMIT_RANGE="HEAD~$1...HEAD" diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 48f81f3dbf..3a04418e8b 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -1,5 +1,7 @@ # ThreadSanitizer suppressions # ============================ +# +# https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions # double locks (TODO fix) mutex:g_genesis_wait_mutex @@ -11,8 +13,9 @@ mutex:CConnman::ThreadOpenConnections mutex:CConnman::ThreadOpenAddedConnections mutex:CConnman::SocketHandler mutex:UpdateTip -mutex:PeerManager::UpdatedBlockTip +mutex:PeerManagerImpl::UpdatedBlockTip mutex:g_best_block_mutex + # race (TODO fix) race:CConnman::WakeMessageHandler race:CConnman::ThreadMessageHandler @@ -27,15 +30,9 @@ race:DatabaseBatch race:leveldb::DBImpl::DeleteObsoleteFiles race:zmq::* race:bitcoin-qt + # deadlock (TODO fix) -deadlock:CConnman::ForNode -deadlock:CConnman::GetNodeStats deadlock:CChainState::ConnectTip -deadlock:UpdateTip -deadlock:wallet_tests::CreateWallet - -# WalletBatch (unidentified deadlock) -deadlock:WalletBatch # Intentional deadlock in tests deadlock:TestPotentialDeadLockDetected @@ -47,4 +44,15 @@ deadlock:src/qt/test/* # External libraries deadlock:libdb race:libzmq -race:epoll_ctl # https://github.com/bitcoin/bitcoin/pull/20218 + +# Intermittent issues +# ------------------- +# +# Suppressions that follow might only happen intermittently, thus they are not +# reproducible. Make sure to include a link to a full trace. + +# https://github.com/bitcoin/bitcoin/issues/20618 +race:CZMQAbstractPublishNotifier::SendZmqMessage + +# https://github.com/bitcoin/bitcoin/pull/20218, https://github.com/bitcoin/bitcoin/pull/20745 +race:epoll_ctl diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index e2f3fec851..9a52cd4b57 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -6,6 +6,7 @@ # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. +unsigned-integer-overflow:*/include/c++/*/bits/basic_string.tcc unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -15,13 +16,7 @@ unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h unsigned-integer-overflow:core_write.cpp -unsigned-integer-overflow:crypto/chacha20.cpp -unsigned-integer-overflow:crypto/ctaes/ctaes.c -unsigned-integer-overflow:crypto/poly1305.cpp -unsigned-integer-overflow:crypto/ripemd160.cpp -unsigned-integer-overflow:crypto/sha1.cpp -unsigned-integer-overflow:crypto/sha256.cpp -unsigned-integer-overflow:crypto/sha512.cpp +unsigned-integer-overflow:crypto/* unsigned-integer-overflow:hash.cpp unsigned-integer-overflow:leveldb/db/log_reader.cc unsigned-integer-overflow:leveldb/util/bloom.cc @@ -76,4 +71,6 @@ implicit-signed-integer-truncation:test/skiplist_tests.cpp implicit-signed-integer-truncation:torcontrol.cpp implicit-unsigned-integer-truncation:crypto/* implicit-unsigned-integer-truncation:leveldb/* +# std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 +implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage implicit-integer-sign-change:crc32c/* diff --git a/test/util/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json index 99cd4ab695..0a9846b4be 100644 --- a/test/util/data/bitcoin-util-test.json +++ b/test/util/data/bitcoin-util-test.json @@ -221,7 +221,7 @@ { "exec": "./bitcoin-tx", "args": ["-create", "outscript=0:123badscript"], "return_code": 1, - "error_txt": "error: script parse error", + "error_txt": "error: script parse error: unknown opcode", "description": "Create a new transaction with an invalid output script" }, { "exec": "./bitcoin-tx", |