diff options
52 files changed, 1534 insertions, 347 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index fde5551531..44b49bb397 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,9 +30,5 @@ task: folder: "/tmp/ccache_dir" depends_built_cache: folder: "/tmp/cirrus-ci-build/depends/built" - install_script: - - apt-get update - - apt-get -y install git bash ccache - - ccache --max-size=${CCACHE_SIZE} ci_script: - ./ci/test_run_all.sh diff --git a/.gitignore b/.gitignore index 5b9ec1f355..23b6090265 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ linux-build win32-build test/config.ini test/cache/* +test/.mypy_cache/ !src/leveldb*/Makefile diff --git a/.travis.yml b/.travis.yml index 1598813ed2..5550de12d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,7 +112,7 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' + name: 'x86_64 Linux [GOAL: install] [focal] [depends, sanitizers: thread (TSan), no wallet, no gui]' # Not enough memory on travis machines, so feature_block is excluded for now env: >- TEST_RUNNER_EXTRA="--exclude feature_block" diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 8b2d609504..26b576c1ae 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -9,6 +9,7 @@ export LC_ALL=C travis_retry pip3 install codespell==1.15.0 travis_retry pip3 install flake8==3.7.8 travis_retry pip3 install yq +travis_retry pip3 install mypy==0.700 SHELLCHECK_VERSION=v0.6.0 curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 63d06dea64..99cf9cd182 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:18.04 -export PACKAGES="clang-9 llvm-9 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" -export NO_DEPENDS=1 +export DOCKER_NAME_TAG=ubuntu:20.04 +export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" +export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-9 CXX=clang++-9" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh index e939b9eeeb..80806aab75 100755 --- a/ci/test/03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -6,10 +6,6 @@ export LC_ALL=C.UTF-8 -# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers. -PATH=$PATH:/usr/lib/llvm-6.0/bin/ -export PATH - BEGIN_FOLD () { echo "" CURRENT_FOLD_NAME=$1 diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 14ab943d68..165983d906 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -24,7 +24,7 @@ mkdir -p "${PREVIOUS_RELEASES_DIR}" export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then @@ -48,16 +48,14 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then --env-file /tmp/env \ --name $CONTAINER_NAME \ $DOCKER_NAME_TAG) - - DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" - } + export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID" else echo "Running on host system without docker wrapper" - DOCKER_EXEC () { - bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" - } fi + +DOCKER_EXEC () { + $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" +} export -f DOCKER_EXEC if [ -n "$DPKG_ADD_ARCH" ]; then diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 3685504524..efaaf154b1 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -33,7 +33,9 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 + python3 -c 'import time; [print(".") or time.sleep(500) for _ in range(4)]' & + ( DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS ) &> /dev/null fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions diff --git a/configure.ac b/configure.ac index 3d8291bf58..eb58eac172 100644 --- a/configure.ac +++ b/configure.ac @@ -391,6 +391,7 @@ if test "x$enable_werror" = "xyes"; then dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) + AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -410,6 +411,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and @@ -1752,7 +1754,7 @@ echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" echo echo " target os = $TARGET_OS" -echo " build os = $BUILD_OS" +echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $CFLAGS" diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index f232bb62c2..d2652119b4 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -1,7 +1,5 @@ -# Valgrind suppressions file for Bitcoin. -# -# Includes known Valgrind warnings in our dependencies that cannot be fixed -# in-tree. +# This valgrind suppressions file includes known Valgrind warnings in our +# dependencies that cannot be fixed in-tree. # # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin @@ -14,6 +12,9 @@ # --error-limit=no src/test/test_bitcoin # # Note that suppressions may depend on OS and/or library versions. +# Tested on: +# * aarch64 (Ubuntu 20.04 system libs, without gui) +# * x86_64 (Ubuntu 18.04 system libs, without gui) { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -47,8 +48,7 @@ Suppress libdb warning Memcheck:Param pwrite64(buf) - fun:pwrite - fun:__os_io + ... obj:*/libdb_cxx-*.so } { diff --git a/doc/release-notes-16528.md b/doc/release-notes-16528.md index 0a7a31cc9d..e69de29bb2 100644 --- a/doc/release-notes-16528.md +++ b/doc/release-notes-16528.md @@ -1,120 +0,0 @@ -Wallet ------- - -### Experimental Descriptor Wallets - -Please note that Descriptor Wallets are still experimental and not all expected functionality -is available. Additionally there may be some bugs and current functions may change in the future. -Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). - -0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store -scriptPubKey information using descriptors. This is in contrast to the Legacy Wallet -structure where keys are used to generate scriptPubKeys and addresses. Because of this -shift to being script based instead of key based, many of the confusing things that Legacy -Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition -of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. -Descriptor Wallets also uses different semantics for watch-only things and imports. - -As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. -Users who already have a Bitcoin Core wallet can continue to use it as they did before without -any change in behavior. Newly created Legacy Wallets (which is the default type of wallet) will -behave as they did in previous versions of Bitcoin Core. - -The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing -things. They are intended to behave similarly except for the import/export and watchonly functionality -as described below. - -#### Creating Descriptor Wallets - -Descriptor Wallets are not created by default. They must be explicitly created using the -`createwallet` RPC or via the GUI. A `descriptors` option has been added to `createwallet`. -Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. - -In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a -Descriptor Wallet should be created. - -Without those options being set, a Legacy Wallet will be created instead. Additionally the -Default Wallet created upon first startup of Bitcoin Core will be a Legacy Wallet. - -#### `IsMine` Semantics - -`IsMine` refers to the function used to determine whether a script belongs to the wallet. -This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets -returns true if the wallet would be able to sign an input that spends an output with that script. -Since keys can be involved in a variety of different scripts, this definition for `IsMine` can -lead to many unexpected scripts being considered part of the wallet. - -With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by -the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly -what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` -in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` -allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type -(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type -even without that address being requested from the wallet. Descriptor Wallets does not -allow for this and will only watch for the addresses that were explicitly requested from the wallet. - -These changes to `IsMine` will make it easier to reason about what scripts the wallet will -actually be watching for in outputs. However for the vast majority of users, this change is -largely transparent and will not have noticeable effect. - -#### Imports and Exports - -In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts -and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` -logic as it has to distinguish between spendable and watchonly. - -Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be -imported. These descriptors are then added to the wallet as if it were a descriptor generated by -the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish -between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also -different and described in more detail in the next section. - -To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax -similar to that of `importmulti`. - -As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys -the existing import RPCs have been disabled for descriptor wallets. -New export RPCs for Descriptor Wallets have not yet been added. - -The following RPCs are disabled for Descriptor Wallets: - -* importprivkey -* importpubkey -* importaddress -* importwallet -* dumpprivkey -* dumpwallet -* importmulti -* addmultisigaddress -* sethdseed - -#### Watchonly Wallets - -A Legacy Wallet contains both private keys and scripts that were being watched. -Those watched scripts would not contribute to your normal balance. In order to see the watchonly -balance and to use watchonly things in transactions, an `include_watchonly` option was added -to many RPCs that would allow users to do that. However it is easy to forget to include this option. - -Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be -watchonly depending on whether it was created with private keys disabled. This eliminates the need -to distinguish between things that are watchonly and things that are not within a wallet itself. - -This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has -a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), -then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT -workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just -non-functional. - -This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such -multiple key descriptors as some transactions could be signed and broadast and others not. This is -due to some transactions containing only single key inputs, while others would contain both single -key and multiple key inputs, depending on which are available and how the coin selection algorithm -selects inputs. However this is not considered to be a supported use case; multisigs -should be in their own wallets which do not already have descriptors. Although users cannot export -descriptors with private keys for now as explained earlier. - -#### BIP 44/49/84 Support - -The change to using descriptors changes the default derivation paths used by Bitcoin Core -to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without -issue. diff --git a/doc/release-notes-18918.md b/doc/release-notes-18918.md index f57a62eeb7..e69de29bb2 100644 --- a/doc/release-notes-18918.md +++ b/doc/release-notes-18918.md @@ -1,3 +0,0 @@ -# Wallet - -The `-salvagewallet` startup option has been removed. A new `salvage` command has been added to the `bitcoin-wallet` tool which performs the salvage operations that `-salvagewallet` did. diff --git a/doc/release-notes.md b/doc/release-notes.md index bcdc591eca..80c385a471 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -112,7 +112,129 @@ Wallet - The wallet can create a transaction without change even when the keypool is empty. Previously it failed. (#17219) -#### Wallet RPC changes +- The `-salvagewallet` startup option has been removed. A new `salvage` command + has been added to the `bitcoin-wallet` tool which performs the salvage + operations that `-salvagewallet` did. (#18918) + +### Experimental Descriptor Wallets + +Please note that Descriptor Wallets are still experimental and not all expected functionality +is available. Additionally there may be some bugs and current functions may change in the future. +Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). + +0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store +scriptPubKey information using descriptors. This is in contrast to the Legacy Wallet +structure where keys are used to generate scriptPubKeys and addresses. Because of this +shift to being script based instead of key based, many of the confusing things that Legacy +Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition +of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. +Descriptor Wallets also uses different semantics for watch-only things and imports. + +As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. +Users who already have a Bitcoin Core wallet can continue to use it as they did before without +any change in behavior. Newly created Legacy Wallets (which is the default type of wallet) will +behave as they did in previous versions of Bitcoin Core. + +The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing +things. They are intended to behave similarly except for the import/export and watchonly functionality +as described below. + +#### Creating Descriptor Wallets + +Descriptor Wallets are not created by default. They must be explicitly created using the +`createwallet` RPC or via the GUI. A `descriptors` option has been added to `createwallet`. +Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. + +In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a +Descriptor Wallet should be created. + +Without those options being set, a Legacy Wallet will be created instead. Additionally the +Default Wallet created upon first startup of Bitcoin Core will be a Legacy Wallet. + +#### `IsMine` Semantics + +`IsMine` refers to the function used to determine whether a script belongs to the wallet. +This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets +returns true if the wallet would be able to sign an input that spends an output with that script. +Since keys can be involved in a variety of different scripts, this definition for `IsMine` can +lead to many unexpected scripts being considered part of the wallet. + +With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by +the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly +what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` +in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` +allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type +(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type +even without that address being requested from the wallet. Descriptor Wallets does not +allow for this and will only watch for the addresses that were explicitly requested from the wallet. + +These changes to `IsMine` will make it easier to reason about what scripts the wallet will +actually be watching for in outputs. However for the vast majority of users, this change is +largely transparent and will not have noticeable effect. + +#### Imports and Exports + +In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts +and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` +logic as it has to distinguish between spendable and watchonly. + +Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be +imported. These descriptors are then added to the wallet as if it were a descriptor generated by +the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish +between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also +different and described in more detail in the next section. + +To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax +similar to that of `importmulti`. + +As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys +the existing import RPCs have been disabled for descriptor wallets. +New export RPCs for Descriptor Wallets have not yet been added. + +The following RPCs are disabled for Descriptor Wallets: + +* importprivkey +* importpubkey +* importaddress +* importwallet +* dumpprivkey +* dumpwallet +* importmulti +* addmultisigaddress +* sethdseed + +#### Watchonly Wallets + +A Legacy Wallet contains both private keys and scripts that were being watched. +Those watched scripts would not contribute to your normal balance. In order to see the watchonly +balance and to use watchonly things in transactions, an `include_watchonly` option was added +to many RPCs that would allow users to do that. However it is easy to forget to include this option. + +Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be +watchonly depending on whether it was created with private keys disabled. This eliminates the need +to distinguish between things that are watchonly and things that are not within a wallet itself. + +This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has +a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), +then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT +workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just +non-functional. + +This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such +multiple key descriptors as some transactions could be signed and broadast and others not. This is +due to some transactions containing only single key inputs, while others would contain both single +key and multiple key inputs, depending on which are available and how the coin selection algorithm +selects inputs. However this is not considered to be a supported use case; multisigs +should be in their own wallets which do not already have descriptors. Although users cannot export +descriptors with private keys for now as explained earlier. + +#### BIP 44/49/84 Support + +The change to using descriptors changes the default derivation paths used by Bitcoin Core +to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without +issue. + +### Wallet RPC changes - The `upgradewallet` RPC replaces the `-upgradewallet` command line option. (#15761) diff --git a/doc/release-notes/release-notes-0.20.0.md b/doc/release-notes/release-notes-0.20.0.md new file mode 100644 index 0000000000..6fc1606b37 --- /dev/null +++ b/doc/release-notes/release-notes-0.20.0.md @@ -0,0 +1,987 @@ +0.20.0 Release Notes +==================== + +Bitcoin Core version 0.20.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.20.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Known Bugs +========== + +The process for generating the source code release ("tarball") has changed in an +effort to make it more complete, however, there are a few regressions in +this release: + +- The generated `configure` script is currently missing, and you will need to + install autotools and run `./autogen.sh` before you can run + `./configure`. This is the same as when checking out from git. + +- Instead of running `make` simply, you should instead run + `BITCOIN_GENBUILD_NO_GIT=1 make`. + +Notable changes +=============== + +P2P and network changes +----------------------- + +#### Removal of BIP61 reject network messages from Bitcoin Core + +The `-enablebip61` command line option to enable BIP61 has been removed. +(#17004) + +This feature has been disabled by default since Bitcoin Core version 0.18.0. +Nodes on the network can not generally be trusted to send valid messages +(including reject messages), so this should only ever be used when +connected to a trusted node. Please use the alternatives recommended +below if you rely on this removed feature: + +- Testing or debugging of implementations of the Bitcoin P2P network protocol + should be done by inspecting the log messages that are produced by a recent + version of Bitcoin Core. Bitcoin Core logs debug messages + (`-debug=<category>`) to a stream (`-printtoconsole`) or to a file + (`-debuglogfile=<debug.log>`). + +- Testing the validity of a block can be achieved by specific RPCs: + + - `submitblock` + + - `getblocktemplate` with `'mode'` set to `'proposal'` for blocks with + potentially invalid POW + +- Testing the validity of a transaction can be achieved by specific RPCs: + + - `sendrawtransaction` + + - `testmempoolaccept` + +- Wallets should not assume a transaction has propagated to the network + just because there are no reject messages. Instead, listen for the + transaction to be announced by other peers on the network. Wallets + should not assume a lack of reject messages means a transaction pays + an appropriate fee. Instead, set fees using fee estimation and use + replace-by-fee to increase a transaction's fee if it hasn't confirmed + within the desired amount of time. + +The removal of BIP61 reject message support also has the following minor RPC +and logging implications: + +- `testmempoolaccept` and `sendrawtransaction` no longer return the P2P reject + code when a transaction is not accepted to the mempool. They still return the + verbal reject reason. + +- Log messages that previously reported the reject code when a transaction was + not accepted to the mempool now no longer report the reject code. The reason + for rejection is still reported. + +Updated RPCs +------------ + +- The RPCs which accept descriptors now accept the new `sortedmulti(...)` descriptor + type which supports multisig scripts where the public keys are sorted + lexicographically in the resulting script. (#17056) + +- The `walletprocesspsbt` and `walletcreatefundedpsbt` RPCs now include + BIP32 derivation paths by default for public keys if we know them. + This can be disabled by setting the `bip32derivs` parameter to + `false`. (#17264) + +- The `bumpfee` RPC's parameter `totalFee`, which was deprecated in + 0.19, has been removed. (#18312) + +- The `bumpfee` RPC will return a PSBT when used with wallets that have + private keys disabled. (#16373) + +- The `getpeerinfo` RPC now includes a `mapped_as` field to indicate the + mapped Autonomous System used for diversifying peer selection. See the + `-asmap` configuration option described below in _New Settings_. (#16702) + +- The `createmultisig` and `addmultisigaddress` RPCs now return an + output script descriptor for the newly created address. (#18032) + +Build System +------------ + +- OpenSSL is no longer used by Bitcoin Core. (#17265) + +- BIP70 support has been fully removed from Bitcoin Core. The + `--enable-bip70` option remains, but it will throw an error during configure. + (#17165) + +- glibc 2.17 or greater is now required to run the release binaries. This + retains compatibility with RHEL 7, CentOS 7, Debian 8 and Ubuntu 14.04 LTS. (#17538) + +- The source code archives that are provided with gitian builds no longer contain + any autotools artifacts. Therefore, to build from such source, a user + should run the `./autogen.sh` script from the root of the unpacked archive. + This implies that `autotools` and other required packages are installed on the + user's system. (#18331) + +New settings +------------ + +- New `rpcwhitelist` and `rpcwhitelistdefault` configuration parameters + allow giving certain RPC users permissions to only some RPC calls. + (#12763) + +- A new `-asmap` configuration option has been added to diversify the + node's network connections by mapping IP addresses Autonomous System + Numbers (ASNs) and then limiting the number of connections made to any + single ASN. See [issue #16599](https://github.com/bitcoin/bitcoin/issues/16599), + [PR #16702](https://github.com/bitcoin/bitcoin/pull/16702), and the + `bitcoind help` for more information. This option is experimental and + subject to removal or breaking changes in future releases, so the + legacy /16 prefix mapping of IP addresses remains the default. (#16702) + +Updated settings +---------------- + +- All custom settings configured when Bitcoin Core starts are now + written to the `debug.log` file to assist troubleshooting. (#16115) + +- Importing blocks upon startup via the `bootstrap.dat` file no longer + occurs by default. The file must now be specified with + `-loadblock=<file>`. (#17044) + +- The `-debug=db` logging category has been renamed to + `-debug=walletdb` to distinguish it from `coindb`. The `-debug=db` + option has been deprecated and will be removed in the next major + release. (#17410) + +- The `-walletnotify` configuration parameter will now replace any `%w` + in its argument with the name of the wallet generating the + notification. This is not supported on Windows. (#13339) + +Removed settings +---------------- + +- The `-whitelistforcerelay` configuration parameter has been removed after + it was discovered that it was rendered ineffective in version 0.13 and + hasn't actually been supported for almost four years. (#17985) + +GUI changes +----------- + +- The "Start Bitcoin Core on system login" option has been removed on macOS. + (#17567) + +- In the Peers window, the details for a peer now displays a `Mapped AS` + field to indicate the mapped Autonomous System used for diversifying + peer selection. See the `-asmap` configuration option in _New + Settings_, above. (#18402) + +- A "known bug" [announced](https://bitcoincore.org/en/releases/0.18.0/#wallet-gui) + in the release notes of version 0.18 has been fixed. The issue + affected anyone who simultaneously used multiple Bitcoin Core wallets + and the GUI coin control feature. (#18894) + +- For watch-only wallets, creating a new transaction in the Send screen + or fee bumping an existing transaction in the Transactions screen will + automatically copy a Partially-Signed Bitcoin Transaction (PSBT) to + the system clipboard. This can then be pasted into an external + program such as [HWI](https://github.com/bitcoin-core/HWI) for + signing. Future versions of Bitcoin Core should support a GUI option + for finalizing and broadcasting PSBTs, but for now the debug console + may be used with the `finalizepsbt` and `sendrawtransaction` RPCs. + (#16944, #17492) + +Wallet +------ + +- The wallet now by default uses bech32 addresses when using RPC, and + creates native segwit change outputs. (#16884) + +- The way that output trust was computed has been fixed, which affects + confirmed/unconfirmed balance status and coin selection. (#16766) + +- The `gettransaction`, `listtransactions` and `listsinceblock` RPC + responses now also include the height of the block that contains the + wallet transaction, if any. (#17437) + +- The `getaddressinfo` RPC has had its `label` field deprecated + (re-enable for this release using the configuration parameter + `-deprecatedrpc=label`). The `labels` field is altered from returning + JSON objects to returning a JSON array of label names (re-enable + previous behavior for this release using the configuration parameter + `-deprecatedrpc=labelspurpose`). Backwards compatibility using the + deprecated configuration parameters is expected to be dropped in the + 0.21 release. (#17585, #17578) + +Documentation changes +--------------------- + +- Bitcoin Core's automatically-generated source code documentation is + now available at https://doxygen.bitcoincore.org. (#17596) + +Low-level changes +================= + +Utilities +--------- + +- The `bitcoin-cli` utility used with the `-getinfo` parameter now + returns a `headers` field with the number of downloaded block headers + on the best headers chain (similar to the `blocks` field that is also + returned) and a `verificationprogress` field that estimates how much + of the best block chain has been synced by the local node. The + information returned no longer includes the `protocolversion`, + `walletversion`, and `keypoololdest` fields. (#17302, #17650) + +- The `bitcoin-cli` utility now accepts a `-stdinwalletpassphrase` + parameter that can be used when calling the `walletpassphrase` and + `walletpassphrasechange` RPCs to read the passphrase from standard + input without echoing it to the terminal, improving security against + anyone who can look at your screen. The existing `-stdinrpcpass` + parameter is also updated to not echo the passphrase. (#13716) + +Command line +------------ + +- Command line options prefixed with main/test/regtest network names like + `-main.port=8333` `-test.server=1` previously were allowed but ignored. Now + they trigger "Invalid parameter" errors on startup. (#17482) + +New RPCs +-------- + +- The `dumptxoutset` RPC outputs a serialized snapshot of the current + UTXO set. A script is provided in the `contrib/devtools` directory + for generating a snapshot of the UTXO set at a particular block + height. (#16899) + +- The `generatetodescriptor` RPC allows testers using regtest mode to + generate blocks that pay an arbitrary output script descriptor. + (#16943) + +Updated RPCs +------------ + +- The `verifychain` RPC default values are now static instead of + depending on the command line options or configuration file + (`-checklevel`, and `-checkblocks`). Users can pass in the RPC + arguments explicitly when they don't want to rely on the default + values. (#18541) + +- The `getblockchaininfo` RPC's `verificationprogress` field will no + longer report values higher than 1. Previously it would occasionally + report the chain was more than 100% verified. (#17328) + +Tests +----- + +- It is now an error to use an unqualified `walletdir=path` setting in + the config file if running on testnet or regtest networks. The setting + now needs to be qualified as `chain.walletdir=path` or placed in the + appropriate `[chain]` section. (#17447) + +- `-fallbackfee` was 0 (disabled) by default for the main chain, but + 0.0002 by default for the test chains. Now it is 0 by default for all + chains. Testnet and regtest users will have to add + `fallbackfee=0.0002` to their configuration if they weren't setting it + and they want it to keep working like before. (#16524) + +Build system +------------ + +- Support is provided for building with the Android Native Development + Kit (NDK). (#16110) + +0.20.0 change log +================= + +### Mining +- #18742 miner: Avoid stack-use-after-return in validationinterface (MarcoFalke) + +### Block and transaction handling +- #15283 log: Fix UB with bench on genesis block (instagibbs) +- #16507 feefilter: Compute the absolute fee rather than stored rate (instagibbs) +- #16688 log: Add validation interface logging (jkczyz) +- #16805 log: Add timing information to FlushStateToDisk() (jamesob) +- #16902 O(1) `OP_IF/NOTIF/ELSE/ENDIF` script implementation (sipa) +- #16945 introduce CChainState::GetCoinsCacheSizeState (jamesob) +- #16974 Walk pindexBestHeader back to ChainActive().Tip() if it is invalid (TheBlueMatt) +- #17004 Remove REJECT code from CValidationState (jnewbery) +- #17080 Explain why `fCheckDuplicateInputs` can not be skipped and remove it (MarcoFalke) +- #17328 GuessVerificationProgress: cap the ratio to 1 (darosior) +- #17399 Templatize ValidationState instead of subclassing (jkczyz) +- #17407 node: Add reference to mempool in NodeContext (MarcoFalke) +- #17708 prevector: Avoid misaligned member accesses (ajtowns) +- #17850,#17896,#17957,#18021,#18021,#18112 Serialization improvements (sipa) +- #17925 Improve UpdateTransactionsFromBlock with Epochs (JeremyRubin) +- #18002 Abstract out script execution out of `VerifyWitnessProgram()` (sipa) +- #18388 Make VerifyWitnessProgram use a Span stack (sipa) +- #18433 serialization: prevent int overflow for big Coin::nHeight (pierreN) +- #18500 chainparams: Bump assumed valid hash (MarcoFalke) +- #18551 Do not clear validationinterface entries being executed (sipa) + +### P2P protocol and network code +- #15437 Remove BIP61 reject messages (MarcoFalke) +- #16702 Supply and use asmap to improve IP bucketing in addrman (naumenkogs) +- #16851 Continue relaying transactions after they expire from mapRelay (ajtowns) +- #17164 Avoid allocating memory for addrKnown where we don't need it (naumenkogs) +- #17243 tools: add PoissonNextSend method that returns mockable time (amitiuttarwar) +- #17251 SocketHandler logs peer id for close and disconnect (Sjors) +- #17573 Seed RNG with precision timestamps on receipt of net messages (TheBlueMatt) +- #17624 Fix an uninitialized read in ProcessMessage(…, "tx", …) when receiving a transaction we already have (practicalswift) +- #17754 Don't allow resolving of std::string with embedded NUL characters. Add tests (practicalswift) +- #17758 Fix CNetAddr::IsRFC2544 comment + tests (tynes) +- #17812 config, net, test: Asmap feature refinements and functional tests (jonatack) +- #17951 Use rolling bloom filter of recent block txs for AlreadyHave() check (sdaftuar) +- #17985 Remove forcerelay of rejected txs (MarcoFalke) +- #18023 Fix some asmap issues (sipa) +- #18054 Reference instead of copy in BlockConnected range loop (jonatack) +- #18376 Fix use-after-free in tests (vasild) +- #18454 Make addr relay mockable, add test (MarcoFalke) +- #18458 Add missing `cs_vNodes` lock (MarcoFalke) +- #18506 Hardcoded seeds update for 0.20 (laanwj) +- #18808 Drop unknown types in getdata (jnewbery) +- #18962 Only send a getheaders for one block in an INV (jnewbery) + +### Wallet +- #13339 Replace %w by wallet name in -walletnotify script (promag) +- #15931 Remove GetDepthInMainChain dependency on locked chain interface (ariard) +- #16373 bumpfee: Return PSBT when wallet has privkeys disabled (instagibbs) +- #16524 Disable -fallbackfee by default (jtimon) +- #16766 Make IsTrusted scan parents recursively (JeremyRubin) +- #16884 Change default address type to bech32 (instagibbs) +- #16911 Only check the hash of transactions loaded from disk (achow101) +- #16923 Handle duplicate fileid exception (promag) +- #17056 descriptors: Introduce sortedmulti descriptor (achow101) +- #17070 Avoid showing GUI popups on RPC errors (MarcoFalke) +- #17138 Remove wallet access to some node arguments (jnewbery) +- #17237 LearnRelatedScripts only if KeepDestination (promag) +- #17260 Split some CWallet functions into new LegacyScriptPubKeyMan (achow101) +- #17261 Make ScriptPubKeyMan an actual interface and the wallet to have multiple (achow101) +- #17290 Enable BnB coin selection for preset inputs and subtract fee from outputs (achow101) +- #17373 Various fixes and cleanup to keypool handling in LegacyScriptPubKeyMan and CWallet (achow101) +- #17410 Rename `db` log category to `walletdb` (like `coindb`) (laanwj) +- #17444 Avoid showing GUI popups on RPC errors (take 2) (MarcoFalke) +- #17447 Make -walletdir network only (promag) +- #17537 Cleanup and move opportunistic and superfluous TopUp()s (achow101) +- #17553 Remove out of date comments for CalculateMaximumSignedTxSize (instagibbs) +- #17568 Fix when sufficient preset inputs and subtractFeeFromOutputs (achow101) +- #17677 Activate watchonly wallet behavior for LegacySPKM only (instagibbs) +- #17719 Document better -keypool as a look-ahead safety mechanism (ariard) +- #17843 Reset reused transactions cache (fjahr) +- #17889 Improve CWallet:MarkDestinationsDirty (promag) +- #18034 Get the OutputType for a descriptor (achow101) +- #18067 Improve LegacyScriptPubKeyMan::CanProvide script recognition (ryanofsky) +- #18115 Pass in transactions and messages for signing instead of exporting the private keys (achow101) +- #18192,#18546 Bugfix: Wallet: Safely deal with change in the address book (luke-jr) +- #18204 descriptors: Improve descriptor cache and cache xpubs (achow101) +- #18274 rpc/wallet: Initialize nFeeRequired to avoid using garbage value on failure (kallewoof) +- #18312 Remove deprecated fee bumping by totalFee (jonatack) +- #18338 Fix wallet unload race condition (promag) + +### RPC and other APIs +- #12763 Add RPC Whitelist Feature from #12248 (JeremyRubin) +- #13716 cli: `-stdinwalletpassphrase` and non-echo stdin passwords (kallewoof) +- #16689 Add missing fields to wallet rpc help output (ariard) +- #16821 Fix bug where duplicate PSBT keys are accepted (erasmospunk) +- #16899 UTXO snapshot creation (dumptxoutset) +- #17156 psbt: Check that various indexes and amounts are within bounds (achow101) +- #17264 Set default bip32derivs to true for psbt methods (Sjors) +- #17283 improve getaddressinfo test coverage, help, code docs (jonatack) +- #17302 cli: Add "headers" and "verificationprogress" to -getinfo (laanwj) +- #17318 replace asserts in RPC code with `CHECK_NONFATAL` and add linter (adamjonas) +- #17437 Expose block height of wallet transactions (promag) +- #17519 Remove unused `COINBASE_FLAGS` (narula) +- #17578 Simplify getaddressinfo labels, deprecate previous behavior (jonatack) +- #17585 deprecate getaddressinfo label (jonatack) +- #17746 Remove vector copy from listtransactions (promag) +- #17809 Auto-format RPCResult (MarcoFalke) +- #18032 Output a descriptor in createmultisig and addmultisigaddress (achow101) +- #18122 Update validateaddress RPCExamples to bech32 (theStack) +- #18208 Change RPCExamples to bech32 (yusufsahinhamza) +- #18268 Remove redundant types from descriptions (docallag) +- #18346 Document an RPCResult for all calls; Enforce at compile time (MarcoFalke) +- #18396 Add missing HelpExampleRpc for getblockfilter (theStack) +- #18398 Fix broken RPCExamples for waitforblock(height) (theStack) +- #18444 Remove final comma for last entry of fixed-size arrays/objects in RPCResult (luke-jr) +- #18459 Remove unused getbalances() code (jonatack) +- #18484 Correctly compute redeemScript from witnessScript for signrawtransaction (achow101) +- #18487 Fix rpcRunLater race in walletpassphrase (promag) +- #18499 Make rpc documentation not depend on call-time rpc args (MarcoFalke) +- #18532 Avoid initialization-order-fiasco on static CRPCCommand tables (MarcoFalke) +- #18541 Make verifychain default values static, not depend on global args (MarcoFalke) +- #18809 Do not advertise dumptxoutset as a way to flush the chainstate (MarcoFalke) +- #18814 Relock wallet only if most recent callback (promag) + +### GUI +- #15023 Restore RPC Console to non-wallet tray icon menu (luke-jr) +- #15084 Don't disable the sync overlay when wallet is disabled (benthecarman) +- #15098 Show addresses for "SendToSelf" transactions (hebasto) +- #15756 Add shortcuts for tab tools (promag) +- #16944 create PSBT with watch-only wallet (Sjors) +- #16964 Change sendcoins dialogue Yes to Send (instagibbs) +- #17068 Always generate `bitcoinstrings.cpp` on `make translate` (D4nte) +- #17096 Rename debug window (Zero-1729) +- #17105 Make RPCConsole::TabTypes an enum class (promag) +- #17125 Add toolTip and placeholderText to sign message fields (dannmat) +- #17165 Remove BIP70 support (fanquake) +- #17180 Improved tooltip for send amount field (JeremyCrookshank) +- #17186 Add placeholder text to the sign message field (Danny-Scott) +- #17195 Send amount placeholder value (JeremyCrookshank) +- #17226 Fix payAmount tooltip in SendCoinsEntry (promag) +- #17360 Cleaning up hide button tool tip (Danny-Scott) +- #17446 Changed tooltip for 'Label' & 'Message' text fields to be more clear (dannmat) +- #17453 Fix intro dialog labels when the prune button is toggled (hebasto) +- #17474 Bugfix: GUI: Recognise `NETWORK_LIMITED` in formatServicesStr (luke-jr) +- #17492 Bump fee returns PSBT on clipboard for watchonly-only wallets (instagibbs) +- #17567 Remove macOS start on login code (fanquake) +- #17587 Show watch-only balance in send screen (Sjors) +- #17694 Disable 3rd-party tx-urls when wallet disabled (brakmic) +- #17696 Force set nPruneSize in QSettings after the intro dialog (hebasto) +- #17702 Move static placeholder texts to forms (laanwj) +- #17826 Log Qt related info (hebasto) +- #17886 Restore English translation option (achow101) +- #17906 Set CConnman byte counters earlier to avoid uninitialized reads (ryanofsky) +- #17935 Hide HD & encryption icons when no wallet loaded (brakmic) +- #17998 Shortcut to close ModalOverlay (emilengler) +- #18007 Bugfix: GUI: Hide the HD/encrypt icons earlier so they get re-shown if another wallet is open (luke-jr) +- #18060 Drop PeerTableModel dependency to ClientModel (promag) +- #18062 Fix unintialized WalletView::progressDialog (promag) +- #18091 Pass clientmodel changes from walletframe to walletviews (jonasschnelli) +- #18101 Fix deprecated QCharRef usage (hebasto) +- #18121 Throttle GUI update pace when -reindex (hebasto) +- #18123 Fix race in WalletModel::pollBalanceChanged (ryanofsky) +- #18160 Avoid Wallet::GetBalance in WalletModel::pollBalanceChanged (promag) +- #18360 Bump transifex slug and update English translations for 0.20 (laanwj) +- #18402 Display mapped AS in peers info window (jonatack) +- #18492 Translations update pre-branch (laanwj) +- #18549 Fix Window -> Minimize menu item (hebasto) +- #18578 Fix leak in CoinControlDialog::updateView (promag) +- #18894 Fix manual coin control with multiple wallets loaded (promag) + +### Build system +- #16667 Remove mingw linker workaround from win gitian descriptor (fanquake) +- #16669 Use new fork of osslsigncode for windows gitian signing (fanquake) +- #16949 Only pass --disable-dependency-tracking to packages that understand it (fanquake) +- #17008 Bump libevent to 2.1.11 in depends (stefanwouldgo) +- #17029 gitian: Various improvements for windows descriptor (dongcarl) +- #17033 Disable _FORTIFY_SOURCE when enable-debug (achow101) +- #17057 Switch to upstream libdmg-hfsplus (fanquake) +- #17066 Remove workaround for ancient libtool (hebasto) +- #17074 Added double quotes (mztriz) +- #17087 Add variable printing target to Makefiles (dongcarl) +- #17118 depends macOS: point --sysroot to SDK (Sjors) +- #17231 Fix boost mac cross build with clang 9+ (theuni) +- #17265 Remove OpenSSL (fanquake) +- #17284 Update retry to current version (RandyMcMillan) +- #17308 nsis: Write to correct filename in first place (dongcarl) +- #17324,#18099 Update univalue subtree (MarcoFalke) +- #17398 Update leveldb to 1.22+ (laanwj) +- #17409 Avoid hardcoded libfaketime dir in gitian (MarcoFalke) +- #17466 Fix C{,XX} pickup (dongcarl) +- #17483 Set gitian arch back to amd64 (MarcoFalke) +- #17486 Make Travis catch unused variables (Sjors) +- #17538 Bump minimum libc to 2.17 for release binaries (fanquake) +- #17542 Create test utility library from src/test/util/ (brakmic) +- #17545 Remove libanl.so.1 from ALLOWED_LIBRARIES (fanquake) +- #17547 Fix configure report about qr (hebasto) +- #17569 Allow export of environ symbols and work around rv64 toolchain issue (laanwj) +- #17647 lcov: filter depends from coverage reports (nijynot) +- #17658 Add ability to skip building qrencode (fanquake) +- #17678 Support for S390X and POWER targets (MarcoFalke) +- #17682 util: Update tinyformat to upstream (laanwj) +- #17698 Don't configure `xcb_proto` (fanquake) +- #17730 Remove Qt networking features (fanquake) +- #17738 Remove linking librt for backwards compatibility (fanquake) +- #17740 Remove configure checks for win libraries we don't link against (fanquake) +- #17741 Included `test_bitcoin-qt` in msvc build (sipsorcery) +- #17756 Remove `WINDOWS_BITS` from build system (fanquake) +- #17769 Set `AC_PREREQ` to 2.69 (fanquake) +- #17880 Add -Wdate-time to Werror flags (fanquake) +- #17910 Remove double `LIBBITCOIN_SERVER` linking (fanquake) +- #17928 Consistent use of package variable (Bushstar) +- #17933 guix: Pin Guix using `guix time-machine` (dongcarl) +- #17948 pass -fno-ident in Windows gitian descriptor (fanquake) +- #18003 Remove --large-address-aware linker flag (fanquake) +- #18004 Don't embed a build-id when building libdmg-hfsplus (fanquake) +- #18051 Fix behavior when `ALLOW_HOST_PACKAGES` unset (hebasto) +- #18059 Add missing attributes to Win installer (fanquake) +- #18104 Skip i686 build by default in guix and gitian (MarcoFalke) +- #18107 Add `cov_fuzz` target (MarcoFalke) +- #18135 Add --enable-determinism configure flag (fanquake) +- #18145 Add Wreturn-type to Werror flags, check on more Travis machines (Sjors) +- #18264 Remove Boost Chrono (fanquake) +- #18290 Set minimum Automake version to 1.13 (hebasto) +- #18320 guix: Remove now-unnecessary gcc make flag (dongcarl) +- #18331 Use git archive as source tarball (hebasto) +- #18397 Fix libevent linking for `bench_bitcoin` binary (hebasto) +- #18426 scripts: `Previous_release`: improve behaviour on failed download (theStack) +- #18429 Remove double `LIBBITCOIN_SERVER` from bench-Makefile (brakmic) +- #18528 Create `test_fuzz` library from src/test/fuzz/fuzz.cpp (brakmic) +- #18558 Fix boost detection for arch armv7l (hebasto) +- #18598 gitian: Add missing automake package to gitian-win-signer.yml (achow101) +- #18676 Check libevent minimum version in configure script (hebasto) +- #18945 Ensure source tarball has leading directory name (laanwj) + +### Platform support +- #16110 Add Android NDK support (icota) +- #16392 macOS toolchain update (fanquake) +- #16569 Increase init file stop timeout (setpill) +- #17151 Remove OpenSSL PRNG seeding (Windows, Qt only) (fanquake) +- #17365 Update README.md with working Android targets and API levels (icota) +- #17521 Only use D-Bus with Qt on linux (fanquake) +- #17550 Set minimum supported macOS to 10.12 (fanquake) +- #17592 Appveyor install libevent[thread] vcpkg (sipsorcery) +- #17660 Remove deprecated key from macOS Info.plist (fanquake) +- #17663 Pass `-dead_strip_dylibs` to ld on macOS (fanquake) +- #17676 Don't use OpenGL in Qt on macOS (fanquake) +- #17686 Add `-bind_at_load` to macOS hardened LDFLAGS (fanquake) +- #17787 scripts: Add macho pie check to security-check.py (fanquake) +- #17800 random: don't special case clock usage on macOS (fanquake) +- #17863 scripts: Add macho dylib checks to symbol-check.py (fanquake) +- #17899 msvc: Ignore msvc linker warning and update to msvc build instructions (sipsorcery) +- #17916 windows: Enable heap terminate-on-corruption (fanquake) +- #18082 logging: Enable `thread_local` usage on macos (fanquake) +- #18108 Fix `.gitignore` policy in `build_msvc` directory (hebasto) +- #18295 scripts: Add macho lazy bindings check to security-check.py (fanquake) +- #18358 util: Fix compilation with mingw-w64 7.0.0 (fanquake) +- #18359 Fix sysctl() detection on macOS (fanquake) +- #18364 random: remove getentropy() fallback for macOS < 10.12 (fanquake) +- #18395 scripts: Add pe dylib checking to symbol-check.py (fanquake) +- #18415 scripts: Add macho tests to test-security-check.py (fanquake) +- #18425 releases: Update with new Windows code signing certificate (achow101) +- #18702 Fix ASLR for bitcoin-cli on Windows (fanquake) + +### Tests and QA +- #12134 Build previous releases and run functional tests (Sjors) +- #13693 Add coverage to estimaterawfee and estimatesmartfee (Empact) +- #13728 lint: Run the ci lint stage on mac (Empact) +- #15443 Add getdescriptorinfo functional test (promag) +- #15888 Add `wallet_implicitsegwit` to test the ability to transform keys between address types (luke-jr) +- #16540 Add `ASSERT_DEBUG_LOG` to unit test framework (MarcoFalke) +- #16597 travis: Run full test suite on native macos (Sjors) +- #16681 Use self.chain instead of 'regtest' in all current tests (jtimon) +- #16786 add unit test for wallet watch-only methods involving PubKeys (theStack) +- #16943 Add generatetodescriptor RPC (MarcoFalke) +- #16973 Fix `combine_logs.py` for AppVeyor build (mzumsande) +- #16975 Show debug log on unit test failure (MarcoFalke) +- #16978 Seed test RNG context for each test case, print seed (MarcoFalke) +- #17009, #17018, #17050, #17051, #17071, #17076, #17083, #17093, #17109, #17113, #17136, #17229, #17291, #17357, #17771, #17777, #17917, #17926, #17972, #17989, #17996, #18009, #18029, #18047, #18126, #18176, #18206, #18353, #18363, #18407, #18417, #18423, #18445, #18455, #18565 Add fuzzing harnesses (practicalswift) +- #17011 ci: Use busybox utils for one build (MarcoFalke) +- #17030 Fix Python Docstring to include all Args (jbampton) +- #17041 ci: Run tests on arm (MarcoFalke) +- #17069 Pass fuzzing inputs as constant references (practicalswift) +- #17091 Add test for loadblock option and linearize scripts (fjahr) +- #17108 fix "tx-size-small" errors after default address change (theStack) +- #17121 Speed up `wallet_backup` by whitelisting peers (immediate tx relay) (theStack) +- #17124 Speed up `wallet_address_types` by whitelisting peers (immediate tx relay) (theStack) +- #17140 Fix bug in `blockfilter_index_tests` (jimpo) +- #17199 use default address type (bech32) for `wallet_bumpfee` tests (theStack) +- #17205 ci: Enable address sanitizer (asan) stack-use-after-return checking (practicalswift) +- #17206 Add testcase to simulate bitcoin schema in leveldb (adamjonas) +- #17209 Remove no longer needed UBSan suppressions (issues fixed). Add documentation (practicalswift) +- #17220 Add unit testing for the CompressScript function (adamjonas) +- #17225 Test serialisation as part of deserialisation fuzzing. Test round-trip equality where possible (practicalswift) +- #17228 Add RegTestingSetup to `setup_common` (MarcoFalke) +- #17233 travis: Run unit and functional tests on native arm (MarcoFalke) +- #17235 Skip unnecessary fuzzer initialisation. Hold ECCVerifyHandle only when needed (practicalswift) +- #17240 ci: Disable functional tests on mac host (MarcoFalke) +- #17254 Fix `script_p2sh_tests` `OP_PUSHBACK2/4` missing (adamjonas) +- #17267 bench: Fix negative values and zero for -evals flag (nijynot) +- #17275 pubkey: Assert CPubKey's ECCVerifyHandle precondition (practicalswift) +- #17288 Added TestWrapper class for interactive Python environments (jachiang) +- #17292 Add new mempool benchmarks for a complex pool (JeremyRubin) +- #17299 add reason checks for non-standard txs in `test_IsStandard` (theStack) +- #17322 Fix input size assertion in `wallet_bumpfee.py` (instagibbs) +- #17327 Add `rpc_fundrawtransaction` logging (jonatack) +- #17330 Add `shrinkdebugfile=0` to regtest bitcoin.conf (sdaftuar) +- #17340 Speed up fundrawtransaction test (jnewbery) +- #17345 Do not instantiate CAddrDB for static call CAddrDB::Read() (hebasto) +- #17362 Speed up `wallet_avoidreuse`, add logging (jonatack) +- #17363 add "diamond" unit test to MempoolAncestryTests (theStack) +- #17366 Reset global args between test suites (MarcoFalke) +- #17367 ci: Run non-cross-compile builds natively (MarcoFalke) +- #17378 TestShell: Fix typos & implement cleanups (jachiang) +- #17384 Create new test library (MarcoFalke) +- #17387 `wallet_importmulti`: use addresses of the same type as being imported (achow101) +- #17388 Add missing newline in `util_ChainMerge` test (ryanofsky) +- #17390 Add `util_ArgParsing` test (ryanofsky) +- #17420 travis: Rework `cache_err_msg` (MarcoFalke) +- #17423 ci: Make ci system read-only on the git work tree (MarcoFalke) +- #17435 check custom ancestor limit in `mempool_packages.py` (theStack) +- #17455 Update valgrind suppressions (practicalswift) +- #17461 Check custom descendant limit in `mempool_packages.py` (theStack) +- #17469 Remove fragile `assert_memory_usage_stable` (MarcoFalke) +- #17470 ci: Use clang-8 for fuzzing to run on aarch64 ci systems (MarcoFalke) +- #17480 Add unit test for non-standard txs with too large scriptSig (theStack) +- #17497 Skip tests when utils haven't been compiled (fanquake) +- #17502 Add unit test for non-standard bare multisig txs (theStack) +- #17511 Add bounds checks before base58 decoding (sipa) +- #17517 ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le (MarcoFalke) +- #17522 Wait until mempool is loaded in `wallet_abandonconflict` (MarcoFalke) +- #17532 Add functional test for non-standard txs with too large scriptSig (theStack) +- #17541 Add functional test for non-standard bare multisig txs (theStack) +- #17555 Add unit test for non-standard txs with wrong nVersion (dspicher) +- #17571 Add `libtest_util` library to msvc build configuration (sipsorcery) +- #17591 ci: Add big endian platform - s390x (elichai) +- #17593 Move more utility functions into test utility library (mzumsande) +- #17633 Add option --valgrind to run the functional tests under Valgrind (practicalswift) +- #17635 ci: Add centos 7 build (hebasto) +- #17641 Add unit test for leveldb creation with unicode path (sipsorcery) +- #17674 Add initialization order fiasco detection in Travis (practicalswift) +- #17675 Enable tests which are incorrectly skipped when running `test_runner.py --usecli` (practicalswift) +- #17685 Fix bug in the descriptor parsing fuzzing harness (`descriptor_parse`) (practicalswift) +- #17705 re-enable CLI test support by using EncodeDecimal in json.dumps() (fanquake) +- #17720 add unit test for non-standard "scriptsig-not-pushonly" txs (theStack) +- #17767 ci: Fix qemu issues (MarcoFalke) +- #17793 ci: Update github actions ci vcpkg cache on msbuild update (hebasto) +- #17806 Change filemode of `rpc_whitelist.py` (emilengler) +- #17849 ci: Fix brew python link (hebasto) +- #17851 Add `std::to_string` to list of locale dependent functions (practicalswift) +- #17893 Fix double-negative arg test (hebasto) +- #17900 ci: Combine 32-bit build with centos 7 build (theStack) +- #17921 Test `OP_CSV` empty stack fail in `feature_csv_activation.py` (theStack) +- #17931 Fix `p2p_invalid_messages` failing in Python 3.8 because of warning (elichai) +- #17947 add unit test for non-standard txs with too large tx size (theStack) +- #17959 Check specific reject reasons in `feature_csv_activation.py` (theStack) +- #17984 Add p2p test for forcerelay permission (MarcoFalke) +- #18001 Updated appveyor job to checkout a specific vcpkg commit ID (sipsorcery) +- #18008 fix fuzzing using libFuzzer on macOS (fanquake) +- #18013 bench: Fix benchmarks filters (elichai) +- #18018 reset fIsBareMultisigStd after bare-multisig tests (fanquake) +- #18022 Fix appveyor `test_bitcoin` build of `*.raw` (MarcoFalke) +- #18037 util: Allow scheduler to be mocked (amitiuttarwar) +- #18056 ci: Check for submodules (emilengler) +- #18069 Replace 'regtest' leftovers by self.chain (theStack) +- #18081 Set a name for CI Docker containers (fanquake) +- #18109 Avoid hitting some known minor tinyformat issues when fuzzing strprintf(…) (practicalswift) +- #18155 Add harness which fuzzes EvalScript and VerifyScript using a fuzzed signature checker (practicalswift) +- #18159 Add --valgrind option to `test/fuzz/test_runner.py` for running fuzzing test cases under valgrind (practicalswift) +- #18166 ci: Run fuzz testing test cases (bitcoin-core/qa-assets) under valgrind to catch memory errors (practicalswift) +- #18172 Transaction expiry from mempool (0xB10C) +- #18181 Remove incorrect assumptions in `validation_flush_tests` (MarcoFalke) +- #18183 Set `catch_system_errors=no` on boost unit tests (MarcoFalke) +- #18195 Add `cost_of_change` parameter assertions to `bnb_search_test` (yancyribbens) +- #18209 Reduce unneeded whitelist permissions in tests (MarcoFalke) +- #18211 Disable mockforward scheduler unit test for now (MarcoFalke) +- #18213 Fix race in `p2p_segwit` (MarcoFalke) +- #18224 Make AnalyzePSBT next role calculation simple, correct (instagibbs) +- #18228 Add missing syncwithvalidationinterfacequeue (MarcoFalke) +- #18247 Wait for both veracks in `add_p2p_connection` (MarcoFalke) +- #18249 Bump timeouts to accomodate really slow disks (MarcoFalke) +- #18255 Add `bad-txns-*-toolarge` test cases to `invalid_txs` (MarcoFalke) +- #18263 rpc: change setmocktime check to use IsMockableChain (gzhao408) +- #18285 Check that `wait_until` returns if time point is in the past (MarcoFalke) +- #18286 Add locale fuzzer to `FUZZERS_MISSING_CORPORA` (practicalswift) +- #18292 fuzz: Add `assert(script == decompressed_script)` (MarcoFalke) +- #18299 Update `FUZZERS_MISSING_CORPORA` to enable regression fuzzing for all harnesses in master (practicalswift) +- #18300 fuzz: Add option to merge input dir to test runner (MarcoFalke) +- #18305 Explain why test logging should be used (MarcoFalke) +- #18306 Add logging to `wallet_listsinceblock.py` (jonatack) +- #18311 Bumpfee test fix (instagibbs) +- #18314 Add deserialization fuzzing of SnapshotMetadata (`utxo_snapshot`) (practicalswift) +- #18319 fuzz: Add missing `ECC_Start` to `key_io` test (MarcoFalke) +- #18334 Add basic test for BIP 37 (MarcoFalke) +- #18350 Fix mining to an invalid target + ensure that a new block has the correct hash internally (TheQuantumPhysicist) +- #18378 Bugfix & simplify bn2vch using `int.to_bytes` (sipa) +- #18393 Don't assume presence of `__builtin_mul_overflow(…)` in `MultiplicationOverflow(…)` fuzzing harness (practicalswift) +- #18406 add executable flag for `rpc_estimatefee.py` (theStack) +- #18420 listsinceblock block height checks (jonatack) +- #18430 ci: Only clone bitcoin-core/qa-assets when fuzzing (MarcoFalke) +- #18438 ci: Use homebrew addon on native macos (hebasto) +- #18447 Add coverage for script parse error in ParseScript (pierreN) +- #18472 Remove unsafe `BOOST_TEST_MESSAGE` (MarcoFalke) +- #18474 check that peer is connected when calling sync_* (MarcoFalke) +- #18477 ci: Use focal for fuzzers (MarcoFalke) +- #18481 add BIP37 'filterclear' test to p2p_filter.py (theStack) +- #18496 Remove redundant `sync_with_ping` after `add_p2p_connection` (jonatack) +- #18509 fuzz: Avoid running over all inputs after merging them (MarcoFalke) +- #18510 fuzz: Add CScriptNum::getint coverage (MarcoFalke) +- #18514 remove rapidcheck integration and tests (fanquake) +- #18515 Add BIP37 remote crash bug [CVE-2013-5700] test to `p2p_filter.py` (theStack) +- #18516 relax bumpfee `dust_to_fee` txsize an extra vbyte (jonatack) +- #18518 fuzz: Extend descriptor fuzz test (MarcoFalke) +- #18519 fuzz: Extend script fuzz test (MarcoFalke) +- #18521 fuzz: Add `process_messages` harness (MarcoFalke) +- #18529 Add fuzzer version of randomized prevector test (sipa) +- #18534 skip backwards compat tests if not compiled with wallet (fanquake) +- #18540 `wallet_bumpfee` assertion fixup (jonatack) +- #18543 Use one node to avoid a race due to missing sync in `rpc_signrawtransaction` (MarcoFalke) +- #18561 Properly raise FailedToStartError when rpc shutdown before warmup finished (MarcoFalke) +- #18562 ci: Run unit tests sequential once (MarcoFalke) +- #18563 Fix `unregister_all_during_call` cleanup (ryanofsky) +- #18566 Set `-use_value_profile=1` when merging fuzz inputs (MarcoFalke) +- #18757 Remove enumeration of expected deserialization exceptions in ProcessMessage(…) fuzzer (practicalswift) +- #18878 Add test for conflicted wallet tx notifications (ryanofsky) +- #18975 Remove const to work around compiler error on xenial (laanwj) + +### Documentation +- #16947 Doxygen-friendly script/descriptor.h comments (ch4ot1c) +- #16983 Add detailed info about Bitcoin Core files (hebasto) +- #16986 Doxygen-friendly CuckooCache comments (ch4ot1c) +- #17022 move-only: Steps for "before major release branch-off" (MarcoFalke) +- #17026 Update bips.md for default bech32 addresses in 0.20.0 (MarcoFalke) +- #17081 Fix Makefile target in benchmarking.md (theStack) +- #17102 Add missing indexes/blockfilter/basic to doc/files.md (MarcoFalke) +- #17119 Fix broken bitcoin-cli examples (andrewtoth) +- #17134 Add switch on enum example to developer notes (hebasto) +- #17142 Update macdeploy README to include all files produced by `make deploy` (za-kk) +- #17146 github: Add warning for bug reports (laanwj) +- #17157 Added instructions for how to add an upsteam to forked repo (dannmat) +- #17159 Add a note about backporting (carnhofdaki) +- #17169 Correct function name in ReportHardwareRand() (fanquake) +- #17177 Describe log files + consistent paths in test READMEs (fjahr) +- #17239 Changed miniupnp links to https (sandakersmann) +- #17281 Add developer note on `c_str()` (laanwj) +- #17285 Bip70 removal follow-up (fjahr) +- #17286 Fix help-debug -checkpoints (ariard) +- #17309 update MSVC instructions to remove Qt OpenSSL linking (fanquake) +- #17339 Add template for good first issues (michaelfolkson) +- #17351 Fix some misspellings (RandyMcMillan) +- #17353 Add ShellCheck to lint tests dependencies (hebasto) +- #17370 Update doc/bips.md with recent changes in master (MarcoFalke) +- #17393 Added regtest config for linearize script (gr0kchain) +- #17411 Add some better examples for scripted diff (laanwj) +- #17503 Remove bitness from bitcoin-qt help message and manpage (laanwj) +- #17539 Update and improve Developer Notes (hebasto) +- #17561 Changed MiniUPnPc link to https in dependencies.md (sandakersmann) +- #17596 Change doxygen URL to doxygen.bitcoincore.org (laanwj) +- #17598 Update release process with latest changes (MarcoFalke) +- #17617 Unify unix epoch time descriptions (jonatack) +- #17637 script: Add keyserver to verify-commits readme (emilengler) +- #17648 Rename wallet-tool references to bitcoin-wallet (hel-o) +- #17688 Add "ci" prefix to CONTRIBUTING.md (hebasto) +- #17751 Use recommended shebang approach in documentation code block (hackerrdave) +- #17752 Fix directory path for secp256k1 subtree in developer-notes (hackerrdave) +- #17772 Mention PR Club in CONTRIBUTING.md (emilengler) +- #17804 Misc RPC help fixes (MarcoFalke) +- #17819 Developer notes guideline on RPCExamples addresses (jonatack) +- #17825 Update dependencies.md (hebasto) +- #17873 Add to Doxygen documentation guidelines (jonatack) +- #17907 Fix improper Doxygen inline comments (Empact) +- #17942 Improve fuzzing docs for macOS users (fjahr) +- #17945 Fix doxygen errors (Empact) +- #18025 Add missing supported rpcs to doc/descriptors.md (andrewtoth) +- #18070 Add note about `brew doctor` (givanse) +- #18125 Remove PPA note from release-process.md (fanquake) +- #18170 Minor grammatical changes and flow improvements (travinkeith) +- #18212 Add missing step in win deployment instructions (dangershony) +- #18219 Add warning against wallet.dat re-use (corollari) +- #18253 Correct spelling errors in comments (Empact) +- #18278 interfaces: Describe and follow some code conventions (ryanofsky) +- #18283 Explain rebase policy in CONTRIBUTING.md (MarcoFalke) +- #18340 Mention MAKE=gmake workaround when building on a BSD (fanquake) +- #18341 Replace remaining literal BTC with `CURRENCY_UNIT` (domob1812) +- #18342 Add fuzzing quickstart guides for libFuzzer and afl-fuzz (practicalswift) +- #18344 Fix nit in getblockchaininfo (stevenroose) +- #18379 Comment fix merkle.cpp (4d55397500) +- #18382 note the costs of fetching all pull requests (vasild) +- #18391 Update init and reduce-traffic docs for -blocksonly (glowang) +- #18464 Block-relay-only vs blocksonly (MarcoFalke) +- #18486 Explain new test logging (MarcoFalke) +- #18505 Update webchat URLs in README.md (SuriyaaKudoIsc) +- #18513 Fix git add argument (HashUnlimited) +- #18577 Correct scripted-diff example link (yahiheb) +- #18589 Fix naming of macOS SDK and clarify version (achow101) + +### Miscellaneous +- #15600 lockedpool: When possible, use madvise to avoid including sensitive information in core dumps (luke-jr) +- #15934 Merge settings one place instead of five places (ryanofsky) +- #16115 On bitcoind startup, write config args to debug.log (LarryRuane) +- #16117 util: Replace boost sleep with std sleep (MarcoFalke) +- #16161 util: Fix compilation errors in support/lockedpool.cpp (jkczyz) +- #16802 scripts: In linearize, search for next position of magic bytes rather than fail (takinbo) +- #16889 Add some general std::vector utility functions (sipa) +- #17049 contrib: Bump gitian descriptors for 0.20 (MarcoFalke) +- #17052 scripts: Update `copyright_header` script to include additional files (GChuf) +- #17059 util: Simplify path argument for cblocktreedb ctor (hebasto) +- #17191 random: Remove call to `RAND_screen()` (Windows only) (fanquake) +- #17192 util: Add `check_nonfatal` and use it in src/rpc (MarcoFalke) +- #17218 Replace the LogPrint function with a macro (jkczyz) +- #17266 util: Rename decodedumptime to parseiso8601datetime (elichai) +- #17270 Feed environment data into RNG initializers (sipa) +- #17282 contrib: Remove accounts from bash completion (fanquake) +- #17293 Add assertion to randrange that input is not 0 (JeremyRubin) +- #17325 log: Fix log message for -par=1 (hebasto) +- #17329 linter: Strip trailing / in path for git-subtree-check (jnewbery) +- #17336 scripts: Search for first block file for linearize-data with some block files pruned (Rjected) +- #17361 scripts: Lint gitian descriptors with shellcheck (hebasto) +- #17482 util: Disallow network-qualified command line options (ryanofsky) +- #17507 random: mark RandAddPeriodic and SeedPeriodic as noexcept (fanquake) +- #17527 Fix CPUID subleaf iteration (sipa) +- #17604 util: Make schedulebatchpriority advisory only (fanquake) +- #17650 util: Remove unwanted fields from bitcoin-cli -getinfo (malevolent) +- #17671 script: Fixed wget call in gitian-build.py (willyko) +- #17699 Make env data logging optional (sipa) +- #17721 util: Don't allow base58 decoding of non-base58 strings. add base58 tests (practicalswift) +- #17750 util: Change getwarnings parameter to bool (jnewbery) +- #17753 util: Don't allow base32/64-decoding or parsemoney(…) on strings with embedded nul characters. add tests (practicalswift) +- #17823 scripts: Read suspicious hosts from a file instead of hardcoding (sanjaykdragon) +- #18162 util: Avoid potential uninitialized read in `formatiso8601datetime(int64_t)` by checking `gmtime_s`/`gmtime_r` return value (practicalswift) +- #18167 Fix a violation of C++ standard rules where unions are used for type-punning (TheQuantumPhysicist) +- #18225 util: Fail to parse empty string in parsemoney (MarcoFalke) +- #18270 util: Fail to parse whitespace-only strings in parsemoney(…) (instead of parsing as zero) (practicalswift) +- #18316 util: Helpexamplerpc formatting (jonatack) +- #18357 Fix missing header in sync.h (promag) +- #18412 script: Fix `script_err_sig_pushonly` error string (theStack) +- #18416 util: Limit decimal range of numbers parsescript accepts (pierreN) +- #18503 init: Replace `URL_WEBSITE` with `PACKAGE_URL` (MarcoFalke) +- #18526 Remove PID file at the very end (hebasto) +- #18553 Avoid non-trivial global constants in SHA-NI code (sipa) +- #18665 Do not expose and consider `-logthreadnames` when it does not work (hebasto) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 251 +- 4d55397500 +- Aaron Clauson +- Adam Jonas +- Albert +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Riard +- Ava Barron +- Ben Carman +- Ben Woosley +- Block Mechanic +- Brian Solon +- Bushstar +- Carl Dong +- Carnhof Daki +- Cory Fields +- Daki Carnhof +- Dan Gershony +- Daniel Kraft +- dannmat +- Danny-Scott +- darosior +- David O'Callaghan +- Dominik Spicher +- Elichai Turkel +- Emil Engler +- emu +- Fabian Jahr +- fanquake +- Filip Gospodinov +- Franck Royer +- Gastón I. Silva +- gchuf +- Gleb Naumenko +- Gloria Zhao +- glowang +- Gr0kchain +- Gregory Sanders +- hackerrdave +- Harris +- hel0 +- Hennadii Stepanov +- ianliu +- Igor Cota +- James Chiang +- James O'Beirne +- Jan Beich +- Jan Sarenik +- Jeffrey Czyz +- Jeremy Rubin +- JeremyCrookshank +- Jim Posen +- John Bampton +- John L. Jegutanis +- John Newbery +- Jon Atack +- Jon Layton +- Jonas Schnelli +- João Barbosa +- Jorge Timón +- Karl-Johan Alm +- kodslav +- Larry Ruane +- Luke Dashjr +- malevolent +- MapleLaker +- marcaiaf +- MarcoFalke +- Marius Kjærstad +- Mark Erhardt +- Mark Tyneway +- Martin Erlandsson +- Martin Zumsande +- Matt Corallo +- Matt Ward +- Michael Folkson +- Michael Polzer +- Micky Yun Chan +- Neha Narula +- nijynot +- naumenkogs +- NullFunctor +- Peter Bushnell +- pierrenn +- Pieter Wuille +- practicalswift +- randymcmillan +- Rjected +- Russell Yanofsky +- Samer Afach +- Samuel Dobson +- Sanjay K +- Sebastian Falbesoner +- setpill +- Sjors Provoost +- Stefan Richter +- stefanwouldgo +- Steven Roose +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Tim Akinbo +- Travin Keith +- tryphe +- Vasil Dimov +- Willy Ko +- Wilson Ccasihue S +- Wladimir J. van der Laan +- Yahia Chiheb +- Yancy Ribbens +- Yusuf Sahin HAMZA +- Zakk +- Zero + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 766c0fca54..93b5156af3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -29,6 +29,7 @@ bench_bench_bitcoin_SOURCES = \ bench/crypto_hash.cpp \ bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ + bench/hashpadding.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp new file mode 100644 index 0000000000..985be8bdba --- /dev/null +++ b/src/bench/hashpadding.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2015-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> + + +static void PrePadded(benchmark::State& state) +{ + + CSHA256 hasher; + + // Setup the salted hasher + uint256 nonce = GetRandHash(); + hasher.Write(nonce.begin(), 32); + hasher.Write(nonce.begin(), 32); + uint256 data = GetRandHash(); + while (state.KeepRunning()) { + unsigned char out[32]; + CSHA256 h = hasher; + h.Write(data.begin(), 32); + h.Finalize(out); + } +} + +BENCHMARK(PrePadded, 10000); + +static void RegularPadded(benchmark::State& state) +{ + CSHA256 hasher; + + // Setup the salted hasher + uint256 nonce = GetRandHash(); + uint256 data = GetRandHash(); + while (state.KeepRunning()) { + unsigned char out[32]; + CSHA256 h = hasher; + h.Write(nonce.begin(), 32); + h.Write(data.begin(), 32); + h.Finalize(out); + } +} + +BENCHMARK(RegularPadded, 10000); diff --git a/src/init.cpp b/src/init.cpp index 37e6251295..0ff89eb816 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -964,17 +964,27 @@ bool AppInitParameterInteraction() // also see: InitParameterInteraction() - // Warn if network-specific options (-addnode, -connect, etc) are + // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden // on the command line or in this network's section of the config file. std::string network = gArgs.GetChainName(); + bilingual_str errors; for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { - return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); + errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); + } + + if (!errors.empty()) { + return InitError(errors); } // Warn if unrecognized section name are present in the config file. + bilingual_str warnings; for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); + warnings += strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized.") + Untranslated("\n"), section.m_file, section.m_line, section.m_name); + } + + if (!warnings.empty()) { + InitWarning(warnings); } if (!fs::is_directory(GetBlocksDir())) { diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index d8e459a8e8..d1e04b114d 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -63,9 +63,9 @@ public: { m_notifications->transactionAddedToMempool(tx); } - void TransactionRemovedFromMempool(const CTransactionRef& tx) override + void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) override { - m_notifications->transactionRemovedFromMempool(tx); + m_notifications->transactionRemovedFromMempool(tx, reason); } void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 7dfc77db7b..61d7ddb934 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -20,6 +20,7 @@ class CRPCCommand; class CScheduler; class Coin; class uint256; +enum class MemPoolRemovalReason; enum class RBFTransactionState; struct bilingual_str; struct CBlockLocator; @@ -239,7 +240,7 @@ public: public: virtual ~Notifications() {} virtual void transactionAddedToMempool(const CTransactionRef& tx) {} - virtual void transactionRemovedFromMempool(const CTransactionRef& ptx) {} + virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} virtual void blockConnected(const CBlock& block, int height) {} virtual void blockDisconnected(const CBlock& block, int height) {} virtual void updatedBlockTip() {} diff --git a/src/net.cpp b/src/net.cpp index 707412bb32..2ccb89248f 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -53,10 +53,17 @@ static constexpr std::chrono::minutes DUMP_PEERS_INTERVAL{15}; static constexpr int DNSSEEDS_TO_QUERY_AT_ONCE = 3; /** How long to delay before querying DNS seeds + * + * If we have more than THRESHOLD entries in addrman, then it's likely + * that we got those addresses from having previously connected to the P2P + * network, and that we'll be able to successfully reconnect to the P2P + * network via contacting one of them. So if that's the case, spend a + * little longer trying to connect to known peers before querying the + * DNS seeds. */ -static constexpr std::chrono::seconds DNSSEEDS_DELAY_FEW_PEERS{11}; // 11sec -static constexpr std::chrono::seconds DNSSEEDS_DELAY_MANY_PEERS{300}; // 5min -static constexpr int DNSSEEDS_DELAY_PEER_THRESHOLD = 1000; // "many" vs "few" peers -- you should only get this many if you've been on the live network +static constexpr std::chrono::seconds DNSSEEDS_DELAY_FEW_PEERS{11}; +static constexpr std::chrono::minutes DNSSEEDS_DELAY_MANY_PEERS{5}; +static constexpr int DNSSEEDS_DELAY_PEER_THRESHOLD = 1000; // "many" vs "few" peers // We add a random period time (0 to 1 seconds) to feeler connections to prevent synchronization. #define FEELER_SLEEP_WINDOW 1 @@ -1595,6 +1602,8 @@ void CConnman::ThreadDNSAddressSeed() seeds_right_now = seeds.size(); } else if (addrman.size() == 0) { // If we have no known peers, query all. + // This will occur on the first run, or if peers.dat has been + // deleted. seeds_right_now = seeds.size(); } @@ -1620,6 +1629,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Waiting %d seconds before querying DNS seeds.\n", seeds_wait_time.count()); std::chrono::seconds to_wait = seeds_wait_time; while (to_wait.count() > 0) { + // if sleeping for the MANY_PEERS interval, wake up + // early to see if we have enough peers and can stop + // this thread entirely freeing up its resources std::chrono::seconds w = std::min(DNSSEEDS_DELAY_FEW_PEERS, to_wait); if (!interruptNet.sleep_for(w)) return; to_wait -= w; @@ -1646,7 +1658,7 @@ void CConnman::ThreadDNSAddressSeed() if (interruptNet) return; - // hold off on querying seeds if p2p network deactivated + // hold off on querying seeds if P2P network deactivated if (!fNetworkActive) { LogPrintf("Waiting for network to be reactivated before querying DNS seeds.\n"); do { @@ -1797,6 +1809,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) return; // Add seed nodes if DNS seeds are all down (an infrastructure attack?). + // Note that we only do this if we started with an empty peers.dat, + // (in which case we will query DNS seeds immediately) *and* the DNS + // seeds have not returned any results. if (addrman.size() == 0 && (GetTime() - nStart > 60)) { static bool done = false; if (!done) { diff --git a/src/protocol.cpp b/src/protocol.cpp index 83a24b9d95..2dfe4bee74 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,8 +5,8 @@ #include <protocol.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #ifndef WIN32 # include <arpa/inet.h> @@ -220,11 +220,7 @@ static std::string serviceFlagToStr(size_t bit) std::ostringstream stream; stream.imbue(std::locale::classic()); stream << "UNKNOWN["; - if (bit < 8) { - stream << service_flag; - } else { - stream << "2^" << bit; - } + stream << "2^" << bit; stream << "]"; return stream.str(); } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index ad74ca3a02..f8cdb5df23 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -65,6 +65,23 @@ Q_DECLARE_METATYPE(CAmount) Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) +static void RegisterMetaTypes() +{ + // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection + qRegisterMetaType<bool*>(); + qRegisterMetaType<SynchronizationState>(); + #ifdef ENABLE_WALLET + qRegisterMetaType<WalletModel*>(); + #endif + // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) + qRegisterMetaType<CAmount>("CAmount"); + qRegisterMetaType<size_t>("size_t"); + + qRegisterMetaType<std::function<void()>>("std::function<void()>"); + qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); +} + static QString GetLangTerritory() { QSettings settings; @@ -184,6 +201,7 @@ BitcoinApplication::BitcoinApplication(interfaces::Node& node): returnValue(0), platformStyle(nullptr) { + RegisterMetaTypes(); setQuitOnLastWindowClosed(false); } @@ -433,20 +451,6 @@ int GuiMain(int argc, char* argv[]) BitcoinApplication app(*node); - // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection - qRegisterMetaType<bool*>(); - qRegisterMetaType<SynchronizationState>(); -#ifdef ENABLE_WALLET - qRegisterMetaType<WalletModel*>(); -#endif - // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) - // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) - qRegisterMetaType<CAmount>("CAmount"); - qRegisterMetaType<size_t>("size_t"); - - qRegisterMetaType<std::function<void()>>("std::function<void()>"); - qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); - /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 2a0079ac39..844f62cbc6 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -22,8 +22,6 @@ static RecursiveMutex cs_rpcWarmup; static std::atomic<bool> g_rpc_running{false}; -static std::once_flag g_rpc_interrupt_flag; -static std::once_flag g_rpc_stop_flag; static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; /* Timer-creating functions */ @@ -295,6 +293,7 @@ void StartRPC() void InterruptRPC() { + static std::once_flag g_rpc_interrupt_flag; // This function could be called twice if the GUI has been started with -server=1. std::call_once(g_rpc_interrupt_flag, []() { LogPrint(BCLog::RPC, "Interrupting RPC\n"); @@ -305,6 +304,7 @@ void InterruptRPC() void StopRPC() { + static std::once_flag g_rpc_stop_flag; // This function could be called twice if the GUI has been started with -server=1. assert(!g_rpc_running); std::call_once(g_rpc_stop_flag, []() { diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index e7b6df3ce8..3c54d5bee4 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -23,7 +23,7 @@ class CSignatureCache { private: //! Entries are SHA256(nonce || signature hash || public key || signature): - uint256 nonce; + CSHA256 m_salted_hasher; typedef CuckooCache::cache<uint256, SignatureCacheHasher> map_type; map_type setValid; boost::shared_mutex cs_sigcache; @@ -31,13 +31,19 @@ private: public: CSignatureCache() { - GetRandBytes(nonce.begin(), 32); + uint256 nonce = GetRandHash(); + // We want the nonce to be 64 bytes long to force the hasher to process + // this chunk, which makes later hash computations more efficient. We + // just write our 32-byte entropy twice to fill the 64 bytes. + m_salted_hasher.Write(nonce.begin(), 32); + m_salted_hasher.Write(nonce.begin(), 32); } void ComputeEntry(uint256& entry, const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubkey) { - CSHA256().Write(nonce.begin(), 32).Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin()); + CSHA256 hasher = m_salted_hasher; + hasher.Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin()); } bool diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index f17b539e09..b4f392116c 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -29,7 +29,6 @@ #endif LockedPoolManager* LockedPoolManager::_instance = nullptr; -std::once_flag LockedPoolManager::init_flag; /*******************************************************************************/ // Utilities diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index de668f0773..b9e2e99d1a 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -221,7 +221,8 @@ public: /** Return the current instance, or create it once */ static LockedPoolManager& Instance() { - std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance); + static std::once_flag init_flag; + std::call_once(init_flag, LockedPoolManager::CreateInstance); return *LockedPoolManager::_instance; } @@ -234,7 +235,6 @@ private: static bool LockingFailed(); static LockedPoolManager* _instance; - static std::once_flag init_flag; }; #endif // BITCOIN_SUPPORT_LOCKEDPOOL_H diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index ca0365d172..211a84b5f2 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -14,6 +14,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/util/mining.h> +#include <test/util/net.h> #include <test/util/setup_common.h> #include <util/memory.h> #include <validationinterface.h> @@ -63,19 +64,23 @@ void initialize() void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; - CNode p2p_node{0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, false}; + CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, false).release(); p2p_node.fSuccessfullyConnected = true; p2p_node.nVersion = PROTOCOL_VERSION; p2p_node.SetSendVersion(PROTOCOL_VERSION); + connman.AddTestNode(p2p_node); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { (void)ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); + LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement + g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index bcbf65bdca..ad6c115a49 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -75,6 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } catch (const std::ios_base::failure&) { } } - connman.ClearTestNodes(); SyncWithValidationInterfaceQueue(); + LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement + g_setup->m_node.connman->StopNodes(); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c5c0208d8f..7d8eb8a323 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -410,7 +410,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) // for any reason except being included in a block. Clients interested // in transactions included in blocks can subscribe to the BlockConnected // notification. - GetMainSignals().TransactionRemovedFromMempool(it->GetSharedTx()); + GetMainSignals().TransactionRemovedFromMempool(it->GetSharedTx(), reason); } const uint256 hash = it->GetTx().GetHash(); diff --git a/src/util/translation.h b/src/util/translation.h index 268bcf30a7..695d6dac96 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -23,6 +23,11 @@ struct bilingual_str { translated += rhs.translated; return *this; } + + bool empty() const + { + return original.empty(); + } }; inline bilingual_str operator+(bilingual_str lhs, const bilingual_str& rhs) diff --git a/src/validation.cpp b/src/validation.cpp index 396fc0a1b5..81b22a7e1f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -50,7 +50,6 @@ #include <string> #include <boost/algorithm/string/replace.hpp> -#include <boost/thread.hpp> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -71,12 +70,12 @@ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; +/** Time to wait between writing blocks/block index to disk. */ +static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; +/** Time to wait between flushing chainstate to disk. */ +static constexpr std::chrono::hours DATABASE_FLUSH_INTERVAL{24}; +/** Maximum age of our tip for us to be considered current for fee estimation */ +static constexpr std::chrono::hours MAX_FEE_ESTIMATION_TIP_AGE{3}; bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... @@ -347,7 +346,7 @@ static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) AssertLockHeld(cs_main); if (::ChainstateActive().IsInitialBlockDownload()) return false; - if (::ChainActive().Tip()->GetBlockTime() < (GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) + if (::ChainActive().Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; if (::ChainActive().Height() < pindexBestHeader->nHeight - 1) return false; @@ -1481,14 +1480,21 @@ int GetSpendHeight(const CCoinsViewCache& inputs) } -static CuckooCache::cache<uint256, SignatureCacheHasher> scriptExecutionCache; -static uint256 scriptExecutionCacheNonce(GetRandHash()); +static CuckooCache::cache<uint256, SignatureCacheHasher> g_scriptExecutionCache; +static CSHA256 g_scriptExecutionCacheHasher; void InitScriptExecutionCache() { + // Setup the salted hasher + uint256 nonce = GetRandHash(); + // We want the nonce to be 64 bytes long to force the hasher to process + // this chunk, which makes later hash computations more efficient. We + // just write our 32-byte entropy twice to fill the 64 bytes. + g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); + g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); - size_t nElems = scriptExecutionCache.setup_bytes(nMaxCacheSize); + size_t nElems = g_scriptExecutionCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for script execution cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); } @@ -1526,12 +1532,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C // properly commits to the scriptPubKey in the inputs view of that // transaction). uint256 hashCacheEntry; - // We only use the first 19 bytes of nonce to avoid a second SHA - // round - giving us 19 + 32 + 4 = 55 bytes (+ 8 + 1 = 64) - static_assert(55 - sizeof(flags) - 32 >= 128/8, "Want at least 128 bits of nonce for script execution cache"); - CSHA256().Write(scriptExecutionCacheNonce.begin(), 55 - sizeof(flags) - 32).Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); + CSHA256 hasher = g_scriptExecutionCacheHasher; + hasher.Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); AssertLockHeld(cs_main); //TODO: Remove this requirement by making CuckooCache not require external locks - if (scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { + if (g_scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { return true; } @@ -1586,7 +1590,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C if (cacheFullScriptStore && !pvChecks) { // We executed all of the provided scripts, and were told to // cache the result. Do so now. - scriptExecutionCache.insert(hashCacheEntry); + g_scriptExecutionCache.insert(hashCacheEntry); } return true; @@ -1765,19 +1769,24 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -void static FlushBlockFile(bool fFinalize = false) +static void FlushUndoFile(int block_file, bool finalize = false) { - LOCK(cs_LastBlockFile); + FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} +static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +{ + LOCK(cs_LastBlockFile); FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); - - bool status = true; - status &= BlockFileSeq().Flush(block_pos_old, fFinalize); - status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); - if (!status) { + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); } static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); @@ -1791,6 +1800,14 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationSt return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) return AbortNode(state, "Failed to write undo data"); + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } // update nUndoPos in block index pindex->nUndoPos = _pos.nPos; @@ -2264,8 +2281,8 @@ bool CChainState::FlushStateToDisk( { LOCK(cs_main); assert(this->CanFlushToDisk()); - static int64_t nLastWrite = 0; - static int64_t nLastFlush = 0; + static std::chrono::microseconds nLastWrite{0}; + static std::chrono::microseconds nLastFlush{0}; std::set<int> setFilesToPrune; bool full_flush_completed = false; @@ -2297,12 +2314,12 @@ bool CChainState::FlushStateToDisk( } } } - int64_t nNow = GetTimeMicros(); + const auto nNow = GetTime<std::chrono::microseconds>(); // Avoid writing/flushing immediately after startup. - if (nLastWrite == 0) { + if (nLastWrite.count() == 0) { nLastWrite = nNow; } - if (nLastFlush == 0) { + if (nLastFlush.count() == 0) { nLastFlush = nNow; } // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). @@ -2310,9 +2327,9 @@ bool CChainState::FlushStateToDisk( // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. - bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + DATABASE_WRITE_INTERVAL; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. - bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + DATABASE_FLUSH_INTERVAL; // Combine all conditions that result in a full cache flush. fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. @@ -2854,8 +2871,6 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar CBlockIndex *pindexNewTip = nullptr; int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { - boost::this_thread::interruption_point(); - // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during // reindex, causing memory blowup if we run too far ahead. @@ -2924,8 +2939,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar // never shutdown before connecting the genesis block during LoadChainTip(). Previously this // caused an assert() failure during shutdown in such cases as the UTXO DB flushing checks // that the best block hash is non-null. - if (ShutdownRequested()) - break; + if (ShutdownRequested()) break; } while (pindexNewTip != pindexMostWork); CheckBlockIndex(chainparams.GetConsensus()); @@ -3243,8 +3257,13 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n vinfoBlockFile.resize(nFile + 1); } + bool finalize_undo = false; if (!fKnown) { while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)ChainActive().Tip()->nHeight); nFile++; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); @@ -3258,7 +3277,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n if (!fKnown) { LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } - FlushBlockFile(!fKnown); + FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; } @@ -4267,7 +4286,6 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ for (pindex = ::ChainActive().Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - boost::this_thread::interruption_point(); const int percentageDone = std::max(1, std::min(99, (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step @@ -4313,8 +4331,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, nGoodTransactions += block.vtx.size(); } } - if (ShutdownRequested()) - return true; + if (ShutdownRequested()) return true; } if (pindexFailure) return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", ::ChainActive().Height() - pindexFailure->nHeight + 1, nGoodTransactions); @@ -4325,7 +4342,6 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { while (pindex != ::ChainActive().Tip()) { - boost::this_thread::interruption_point(); const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); if (reportDone < percentageDone/10) { // report every 10% step @@ -4339,6 +4355,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); + if (ShutdownRequested()) return true; } } diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 9437f9c817..3dfbcc581c 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -199,22 +199,22 @@ void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockInd fInitialDownload); } -void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - auto event = [ptx, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(ptx); }); +void CMainSignals::TransactionAddedToMempool(const CTransactionRef& tx) { + auto event = [tx, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(tx); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, - ptx->GetHash().ToString(), - ptx->GetWitnessHash().ToString()); + tx->GetHash().ToString(), + tx->GetWitnessHash().ToString()); } -void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { - auto event = [ptx, this] { - m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(ptx); }); +void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { + auto event = [tx, reason, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(tx, reason); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, - ptx->GetHash().ToString(), - ptx->GetWitnessHash().ToString()); + tx->GetHash().ToString(), + tx->GetWitnessHash().ToString()); } void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { diff --git a/src/validationinterface.h b/src/validationinterface.h index 9c23965bc1..e96f2883fc 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -21,6 +21,7 @@ class CConnman; class CValidationInterface; class uint256; class CScheduler; +enum class MemPoolRemovalReason; /** Register subscriber */ void RegisterValidationInterface(CValidationInterface* callbacks); @@ -96,7 +97,7 @@ protected: * * Called on a background thread. */ - virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} + virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} /** * Notifies listeners of a transaction leaving mempool. * @@ -129,7 +130,7 @@ protected: * * Called on a background thread. */ - virtual void TransactionRemovedFromMempool(const CTransactionRef &ptx) {} + virtual void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. @@ -196,8 +197,8 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); - void TransactionAddedToMempool(const CTransactionRef &); - void TransactionRemovedFromMempool(const CTransactionRef &); + void TransactionAddedToMempool(const CTransactionRef&); + void TransactionRemovedFromMempool(const CTransactionRef&, MemPoolRemovalReason); void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex); void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(const CBlockLocator &); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 4ed28b0623..d90e8e6433 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -14,8 +14,6 @@ #include <sys/stat.h> #endif -#include <boost/thread.hpp> - namespace { //! Make sure database has a unique fileid within the environment. If it @@ -671,7 +669,6 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) if (nRefCount == 0) { - boost::this_thread::interruption_point(); std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 7824563254..89737ca7b5 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -21,6 +21,7 @@ #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> +#include <txmempool.h> #include <util/bip32.h> #include <util/check.h> #include <util/error.h> @@ -1100,23 +1101,52 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio MarkInputsDirty(ptx); } -void CWallet::transactionAddedToMempool(const CTransactionRef& ptx) { +void CWallet::transactionAddedToMempool(const CTransactionRef& tx) { LOCK(cs_wallet); - CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); - SyncTransaction(ptx, confirm); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); - auto it = mapWallet.find(ptx->GetHash()); + auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { it->second.fInMempool = true; } } -void CWallet::transactionRemovedFromMempool(const CTransactionRef &ptx) { +void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) { LOCK(cs_wallet); - auto it = mapWallet.find(ptx->GetHash()); + auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { it->second.fInMempool = false; } + // Handle transactions that were removed from the mempool because they + // conflict with transactions in a newly connected block. + if (reason == MemPoolRemovalReason::CONFLICT) { + // Call SyncNotifications, so external -walletnotify notifications will + // be triggered for these transactions. Set Status::UNCONFIRMED instead + // of Status::CONFLICTED for a few reasons: + // + // 1. The transactionRemovedFromMempool callback does not currently + // provide the conflicting block's hash and height, and for backwards + // compatibility reasons it may not be not safe to store conflicted + // wallet transactions with a null block hash. See + // https://github.com/bitcoin/bitcoin/pull/18600#discussion_r420195993. + // 2. For most of these transactions, the wallet's internal conflict + // detection in the blockConnected handler will subsequently call + // MarkConflicted and update them with CONFLICTED status anyway. This + // applies to any wallet transaction that has inputs spent in the + // block, or that has ancestors in the wallet with inputs spent by + // the block. + // 3. Longstanding behavior since the sync implementation in + // https://github.com/bitcoin/bitcoin/pull/9371 and the prior sync + // implementation before that was to mark these transactions + // unconfirmed rather than conflicted. + // + // Nothing described above should be seen as an unchangeable requirement + // when improving this code in the future. The wallet's heuristics for + // distinguishing between conflicted and unconfirmed transactions are + // imperfect, and could be improved in general, see + // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + } } void CWallet::blockConnected(const CBlock& block, int height) @@ -1127,9 +1157,8 @@ void CWallet::blockConnected(const CBlock& block, int height) m_last_block_processed_height = height; m_last_block_processed = block_hash; for (size_t index = 0; index < block.vtx.size(); index++) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, block_hash, index); - SyncTransaction(block.vtx[index], confirm); - transactionRemovedFromMempool(block.vtx[index]); + SyncTransaction(block.vtx[index], {CWalletTx::Status::CONFIRMED, height, block_hash, (int)index}); + transactionRemovedFromMempool(block.vtx[index], MemPoolRemovalReason::BLOCK); } } @@ -1144,8 +1173,7 @@ void CWallet::blockDisconnected(const CBlock& block, int height) m_last_block_processed_height = height - 1; m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); - SyncTransaction(ptx, confirm); + SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); } } @@ -1685,8 +1713,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block_height, block_hash, posInBlock); - SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); + SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e3141baef0..67331dc3be 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -921,7 +921,7 @@ public: uint256 last_failed_block; }; ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); - void transactionRemovedFromMempool(const CTransactionRef &ptx) override; + void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); struct Balance { diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index e7adbfea77..cb516f70f0 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -18,8 +18,6 @@ #include <atomic> #include <string> -#include <boost/thread.hpp> - namespace DBKeys { const std::string ACENTRY{"acentry"}; const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; @@ -745,11 +743,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) pwallet->WalletLogPrintf("%s\n", strErr); } pcursor->close(); - } - catch (const boost::thread_interrupted&) { - throw; - } - catch (...) { + } catch (...) { result = DBErrors::CORRUPT; } @@ -887,11 +881,7 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal } } pcursor->close(); - } - catch (const boost::thread_interrupted&) { - throw; - } - catch (...) { + } catch (...) { result = DBErrors::CORRUPT; } diff --git a/test/functional/README.md b/test/functional/README.md index 004e0afb1d..aff5f714f2 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -26,10 +26,12 @@ don't have test cases for. The Travis linter also checks this, but [possibly not in all cases](https://github.com/bitcoin/bitcoin/pull/14884#discussion_r239585126). - See [the python lint script](/test/lint/lint-python.sh) that checks for violations that could lead to bugs and issues in the test code. +- Use [type hints](https://docs.python.org/3/library/typing.html) in your code to improve code readability + and to detect possible bugs earlier. - Avoid wildcard imports - Use a module-level docstring to describe what the test is testing, and how it is testing it. -- When subclassing the BitcoinTestFramwork, place overrides for the +- When subclassing the BitcoinTestFramework, place overrides for the `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of the subclass, then locally-defined helper methods, then the `run_test()` method. - Use `'{}'.format(x)` for string formatting, not `'%s' % x`. @@ -45,7 +47,7 @@ don't have test cases for. - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` -- use an underscore to separate words +- Use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` - Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py` diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index ae5721bec2..6e72db1d96 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -21,6 +21,7 @@ Invalid tx cases not covered here can be found by running: """ import abc +from typing import Optional from test_framework.messages import ( COutPoint, CTransaction, @@ -56,7 +57,7 @@ class BadTxTemplate: __metaclass__ = abc.ABCMeta # The expected error code given by bitcoind upon submission of the tx. - reject_reason = "" + reject_reason = "" # type: Optional[str] # Only specified if it differs from mempool acceptance error. block_reject_reason = "" diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 1a7c656274..a4dc455d57 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -71,7 +71,7 @@ class ConfArgsTest(BitcoinTestFramework): with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 47200b6cc6..fb0c7ceed4 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -125,12 +125,7 @@ class NotificationsTest(BitcoinTestFramework): # Bump tx2 as bump2 and generate a block on node 0 while # disconnected, then reconnect and check for notifications on node 1 - # about newly confirmed bump2 and newly conflicted tx2. Currently - # only the bump2 notification is sent. Ideally, notifications would - # be sent both for bump2 and tx2, which was the previous behavior - # before being broken by an accidental change in PR - # https://github.com/bitcoin/bitcoin/pull/16624. The bug is reported - # in issue https://github.com/bitcoin/bitcoin/issues/18325. + # about newly confirmed bump2 and newly conflicted tx2. disconnect_nodes(self.nodes[0], 1) bump2 = self.nodes[0].bumpfee(tx2)["txid"] self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) @@ -138,7 +133,7 @@ class NotificationsTest(BitcoinTestFramework): assert_equal(tx2 in self.nodes[1].getrawmempool(), True) connect_nodes(self.nodes[0], 1) self.sync_blocks() - self.expect_wallet_notify([bump2]) + self.expect_wallet_notify([bump2, tx2]) assert_equal(self.nodes[1].gettransaction(bump2)["confirmations"], 1) # TODO: add test for `-alertnotify` large fork notifications diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 1bda167c87..86d7c78d63 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -29,8 +29,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, ) -from test_framework.script import CScriptNum - def assert_template(node, block, expect, rehash=True): if rehash: @@ -91,12 +89,6 @@ class MiningTest(BitcoinTestFramework): coinbase_tx.rehash() # round-trip the encoded bip34 block height commitment - assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), next_height) - # round-trip negative and multi-byte CScriptNums to catch python regression - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(1500))), 1500) - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1500))), -1500) - assert_equal(CScriptNum.decode(CScriptNum.encode(CScriptNum(-1))), -1) - block = CBlock() block.nVersion = tmpl["version"] block.hashPrevBlock = int(tmpl["previousblockhash"], 16) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8a989097b4..8803086213 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -295,7 +295,7 @@ class SegWitTest(BitcoinTestFramework): return func_wrapper - @subtest + @subtest # type: ignore def test_non_witness_transaction(self): """See if sending a regular transaction works, and create a utxo to use in later tests.""" # Mine a block with an anyone-can-spend coinbase, @@ -324,7 +324,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) - @subtest + @subtest # type: ignore def test_unnecessary_witness_before_segwit_activation(self): """Verify that blocks with witnesses are rejected before activation.""" @@ -355,7 +355,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_block_relay(self): """Test that block requests to NODE_WITNESS peer are with MSG_WITNESS_FLAG. @@ -451,7 +451,7 @@ class SegWitTest(BitcoinTestFramework): self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert block4.sha256 not in self.old_node.getdataset - @subtest + @subtest # type: ignore def test_v0_outputs_arent_spendable(self): """Test that v0 outputs aren't spendable before segwit activation. @@ -533,7 +533,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) - @subtest + @subtest # type: ignore def test_getblocktemplate_before_lockin(self): txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) @@ -559,7 +559,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_blocks() - @subtest + @subtest # type: ignore def test_witness_tx_relay_before_segwit_activation(self): # Generate a transaction that doesn't require a witness, but send it @@ -601,7 +601,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx_hash, 0, tx_value)) - @subtest + @subtest # type: ignore def test_standardness_v0(self): """Test V0 txout standardness. @@ -679,7 +679,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) assert_equal(len(self.nodes[1].getrawmempool()), 0) - @subtest + @subtest # type: ignore def advance_to_segwit_active(self): """Mine enough blocks to activate segwit.""" assert not softfork_active(self.nodes[0], 'segwit') @@ -690,7 +690,7 @@ class SegWitTest(BitcoinTestFramework): assert softfork_active(self.nodes[0], 'segwit') self.segwit_active = True - @subtest + @subtest # type: ignore def test_p2sh_witness(self): """Test P2SH wrapped witness programs.""" @@ -759,7 +759,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(spend_tx.sha256, 0, spend_tx.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_witness_commitments(self): """Test witness commitments. @@ -849,7 +849,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_block_malleability(self): # Make sure that a block that has too big a virtual size @@ -889,7 +889,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - @subtest + @subtest # type: ignore def test_witness_block_size(self): # TODO: Test that non-witness carrying blocks can't exceed 1MB # Skipping this test for now; this is covered in p2p-fullblocktest.py @@ -967,7 +967,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue)) - @subtest + @subtest # type: ignore def test_submit_block(self): """Test that submitblock adds the nonce automatically when possible.""" block = self.build_next_block() @@ -1003,7 +1003,7 @@ class SegWitTest(BitcoinTestFramework): # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash - @subtest + @subtest # type: ignore def test_extra_witness_data(self): """Test extra witness data in a transaction.""" @@ -1076,7 +1076,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_max_witness_push_length(self): """Test that witness stack can only allow up to 520 byte pushes.""" @@ -1113,7 +1113,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_max_witness_program_length(self): """Test that witness outputs greater than 10kB can't be spent.""" @@ -1161,7 +1161,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_witness_input_length(self): """Test that vin length must match vtxinwit length.""" @@ -1243,7 +1243,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_tx_relay_after_segwit_activation(self): """Test transaction relay after segwit activation. @@ -1336,7 +1336,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_segwit_versions(self): """Test validity of future segwit version transactions. @@ -1418,7 +1418,7 @@ class SegWitTest(BitcoinTestFramework): # Add utxo to our list self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_premature_coinbase_witness_spend(self): block = self.build_next_block() @@ -1453,7 +1453,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) self.sync_blocks() - @subtest + @subtest # type: ignore def test_uncompressed_pubkey(self): """Test uncompressed pubkey validity in segwit transactions. @@ -1558,7 +1558,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block, accepted=True) self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_signature_version_1(self): key = ECKey() @@ -1740,7 +1740,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(len(tx.vout)): self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) - @subtest + @subtest # type: ignore def test_non_standard_witness_blinding(self): """Test behavior of unnecessary witnesses in transactions does not blind the node for the transaction""" @@ -1794,7 +1794,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest + @subtest # type: ignore def test_non_standard_witness(self): """Test detection of non-standard P2WSH witness""" pad = chr(1).encode('latin-1') @@ -1894,7 +1894,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) - @subtest + @subtest # type: ignore def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" @@ -1916,7 +1916,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[0].getblock(block_hash), self.nodes[2].getblock(block_hash)) height -= 1 - @subtest + @subtest # type: ignore def test_witness_sigops(self): """Test sigop counting is correct inside witnesses.""" diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 376bb35f07..58d8c4abe1 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -28,6 +28,7 @@ from test_framework.messages import ( NODE_WITNESS, ) + def assert_net_servicesnames(servicesflag, servicenames): """Utility that checks if all flags are correctly decoded in `getpeerinfo` and `getnetworkinfo`. @@ -40,6 +41,7 @@ def assert_net_servicesnames(servicesflag, servicenames): servicesflag_generated |= getattr(test_framework.messages, 'NODE_' + servicename) assert servicesflag_generated == servicesflag + class NetTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -57,6 +59,7 @@ class NetTest(BitcoinTestFramework): self._test_getnetworkinfo() self._test_getaddednodeinfo() self._test_getpeerinfo() + self.test_service_flags() self._test_getnodeaddresses() def _test_connection_count(self): @@ -139,6 +142,11 @@ class NetTest(BitcoinTestFramework): for info in peer_info: assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) + def test_service_flags(self): + self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63)) + assert_equal(['UNKNOWN[2^4]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) + self.nodes[0].disconnect_p2ps() + def _test_getnodeaddresses(self): self.nodes[0].add_p2p_connection(P2PInterface()) @@ -174,5 +182,6 @@ class NetTest(BitcoinTestFramework): node_addresses = self.nodes[0].getnodeaddresses(LARGE_REQUEST_COUNT) assert_greater_than(LARGE_REQUEST_COUNT, len(node_addresses)) + if __name__ == '__main__': NetTest().main() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index d741b00ba0..afc1995009 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -4,6 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" +import unittest + from .address import ( key_to_p2sh_p2wpkh, key_to_p2wpkh, @@ -217,3 +219,9 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = ToHex(tx) return node.sendrawtransaction(tx_to_witness) + +class TestFrameworkBlockTools(unittest.TestCase): + def test_create_coinbase(self): + height = 20 + coinbase_tx = create_coinbase(height=height) + assert_equal(CScriptNum.decode(coinbase_tx.vin[0].scriptSig), height) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 9102266456..cc5f8307d3 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -9,6 +9,7 @@ This file is modified from python-bitcoinlib. import hashlib import struct import unittest +from typing import List, Dict from .messages import ( CTransaction, @@ -21,7 +22,7 @@ from .messages import ( ) MAX_SCRIPT_ELEMENT_SIZE = 520 -OPCODE_NAMES = {} +OPCODE_NAMES = {} # type: Dict[CScriptOp, str] def hash160(s): return hashlib.new('ripemd160', sha256(s)).digest() @@ -37,7 +38,7 @@ def bn2vch(v): # Serialize to bytes return encoded_v.to_bytes(n_bytes, 'little') -_opcode_instances = [] +_opcode_instances = [] # type: List[CScriptOp] class CScriptOp(int): """A single script opcode""" __slots__ = () @@ -731,3 +732,9 @@ class TestFrameworkScript(unittest.TestCase): self.assertEqual(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) self.assertEqual(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) self.assertEqual(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) + + def test_cscriptnum_encoding(self): + # round-trip negative and multi-byte CScriptNums + values = [0, 1, -1, -2, 127, 128, -255, 256, (1 << 15) - 1, -(1 << 16), (1 << 24) - 1, (1 << 31), 1 - (1 << 32), 1 << 40, 1500, -1500] + for value in values: + self.assertEqual(CScriptNum.decode(CScriptNum.encode(CScriptNum(value))), value) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 5469f808d1..8e0da5295d 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -91,6 +91,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" + chain = None # type: str + setup_clean_chain = None # type: bool + def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" self.chain = 'regtest' @@ -407,7 +410,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Public helper methods. These can be accessed by the subclass test scripts. - def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None, binary_cli=None, versions=None): + def add_nodes(self, num_nodes: int, extra_args=None, *, rpchost=None, binary=None, binary_cli=None, versions=None): """Instantiate TestNode objects. Should only be called once after the nodes have been specified in diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 0812470b0c..0ea65c68b8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -42,7 +42,7 @@ except UnicodeDecodeError: if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): if os.name == 'nt': import ctypes - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -68,6 +68,7 @@ TEST_EXIT_SKIPPED = 77 TEST_FRAMEWORK_MODULES = [ "address", + "blocktools", "script", ] diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 5b083a5398..c441b75652 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -11,7 +11,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - assert_raises_rpc_error + assert_raises_rpc_error, ) @@ -32,11 +32,11 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].getaddressinfo(change_addr) + change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/0") else: - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key # Import a non-HD private key in the HD wallet non_hd_add = 'bcrt1qmevj8zfx0wdvp05cqwkmr6mxkfx60yezwjksmt' @@ -58,7 +58,7 @@ class WalletHDTest(BitcoinTestFramework): if self.options.descriptors: assert_equal(hd_info["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) else: - assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) @@ -67,11 +67,11 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].getaddressinfo(change_addr) + change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/1") else: - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key self.sync_all() assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) @@ -82,7 +82,10 @@ class WalletHDTest(BitcoinTestFramework): # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat")) + shutil.copyfile( + os.path.join(self.nodes[1].datadir, "hd.bak"), + os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat"), + ) self.start_node(1) # Assert that derivation is deterministic @@ -93,7 +96,7 @@ class WalletHDTest(BitcoinTestFramework): if self.options.descriptors: assert_equal(hd_info_2["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) else: - assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info_2["hdmasterfingerprint"], hd_fingerprint) assert_equal(hd_add, hd_add_2) connect_nodes(self.nodes[0], 1) @@ -108,7 +111,10 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) + shutil.copyfile( + os.path.join(self.nodes[1].datadir, "hd.bak"), + os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat"), + ) self.start_node(1, extra_args=self.extra_args[1]) connect_nodes(self.nodes[0], 1) self.sync_all() @@ -142,8 +148,9 @@ class WalletHDTest(BitcoinTestFramework): new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] assert orig_masterkeyid != new_masterkeyid addr = self.nodes[1].getnewaddress() - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool - self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key + # Make sure the new address is the first from the keypool + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') + self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key # Set a new HD seed on node 1 without flushing the keypool new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) @@ -153,13 +160,15 @@ class WalletHDTest(BitcoinTestFramework): assert orig_masterkeyid != new_masterkeyid addr = self.nodes[1].getnewaddress() assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool + # Make sure the new address continues previous keypool + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Check that the next address is from the new seed self.nodes[1].keypoolrefill(1) next_addr = self.nodes[1].getnewaddress() assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool + # Make sure the new address is not from previous keypool + assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') assert next_addr != addr # Sethdseed parameter validity @@ -185,13 +194,13 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name='restore', blank=True) restore_rpc = self.nodes[1].get_wallet_rpc('restore') - restore_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc - restore_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + restore_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive self.nodes[1].createwallet(wallet_name='restore2', blank=True) restore2_rpc = self.nodes[1].get_wallet_rpc('restore2') - restore2_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc - restore2_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + restore2_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore2_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive # Check persistence of inactive seed by reloading restore. restore2 is still loaded to test the case where the wallet is not reloaded restore_rpc.unloadwallet() @@ -201,8 +210,8 @@ class WalletHDTest(BitcoinTestFramework): # Empty origin keypool and get an address that is beyond the initial keypool origin_rpc.getnewaddress() origin_rpc.getnewaddress() - last_addr = origin_rpc.getnewaddress() # Last address of initial keypool - addr = origin_rpc.getnewaddress() # First address beyond initial keypool + last_addr = origin_rpc.getnewaddress() # Last address of initial keypool + addr = origin_rpc.getnewaddress() # First address beyond initial keypool # Check that the restored seed has last_addr but does not have addr info = restore_rpc.getaddressinfo(last_addr) @@ -222,6 +231,7 @@ class WalletHDTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) self.nodes[0].generate(1) + self.sync_blocks() origin_rpc.gettransaction(txid) assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid) out_of_kp_txid = txid @@ -232,6 +242,7 @@ class WalletHDTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(last_addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) self.nodes[0].generate(1) + self.sync_blocks() origin_rpc.gettransaction(txid) restore_rpc.gettransaction(txid) assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, out_of_kp_txid) @@ -266,5 +277,6 @@ class WalletHDTest(BitcoinTestFramework): info = restore2_rpc.getaddressinfo(addr) assert_equal(info['ismine'], False) + if __name__ == '__main__': - WalletHDTest().main () + WalletHDTest().main() diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index f8d1720469..fb4a1f9792 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -134,6 +134,33 @@ class WalletLabelsTest(BitcoinTestFramework): # in the label. This is a no-op. change_label(node, labels[2].addresses[0], labels[2], labels[2]) + self.log.info('Check watchonly labels') + node.createwallet(wallet_name='watch_only', disable_private_keys=True, descriptors=False) + wallet_watch_only = node.get_wallet_rpc('watch_only') + BECH32_VALID = { + '✔️_VER15_PROG40': 'bcrt10qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqn2cjv3', + '✔️_VER16_PROG03': 'bcrt1sqqqqqjq8pdp', + '✔️_VER16_PROB02': 'bcrt1sqqqqqjq8pv', + } + BECH32_INVALID = { + '❌_VER15_PROG41': 'bcrt10qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqzc7xyq', + '❌_VER16_PROB01': 'bcrt1sqqpl9r5c', + } + for l in BECH32_VALID: + ad = BECH32_VALID[l] + wallet_watch_only.importaddress(label=l, rescan=False, address=ad) + node.generatetoaddress(1, ad) + assert_equal(wallet_watch_only.getaddressesbylabel(label=l), {ad: {'purpose': 'receive'}}) + assert_equal(wallet_watch_only.getreceivedbylabel(label=l), 0) + for l in BECH32_INVALID: + ad = BECH32_INVALID[l] + assert_raises_rpc_error( + -5, + "Invalid Bitcoin address or script", + lambda: wallet_watch_only.importaddress(label=l, rescan=False, address=ad), + ) + + class Label: def __init__(self, name): # Label name diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 86ac5a930f..b9aa6c799b 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -7,6 +7,7 @@ # Check for specified flake8 warnings in python files. export LC_ALL=C +export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" enabled=( E101 # indentation contains mixed spaces and tabs @@ -96,3 +97,5 @@ PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo " echo "$@" fi ) + +mypy --ignore-missing-imports $(git ls-files "test/functional/*.py")
\ No newline at end of file diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 70eea34363..f71599d92f 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -1,6 +1,29 @@ # ThreadSanitizer suppressions # ============================ +# double locks (TODO fix) +mutex:g_genesis_wait_mutex +mutex:Interrupt +mutex:CThreadInterrupt +mutex:CConnman::Interrupt +mutex:CConnman::WakeMessageHandler +mutex:CConnman::ThreadOpenConnections +mutex:CConnman::ThreadOpenAddedConnections +mutex:CConnman::SocketHandler +mutex:UpdateTip +mutex:PeerLogicValidation::UpdatedBlockTip +# race (TODO fix) +race:CConnman::WakeMessageHandler +race:CConnman::ThreadMessageHandler +race:fHaveGenesis +race:ProcessNewBlock +race:ThreadImport +race:zmq::* +race:bitcoin-qt +# deadlock (TODO fix) +deadlock:CConnman::ForNode +deadlock:UpdateTip + # WalletBatch (unidentified deadlock) deadlock:WalletBatch |