diff options
484 files changed, 10415 insertions, 7687 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 28c1814998..55cebc008f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,37 +1,21 @@ -<!--- Remove sections that do not apply --> +<!-- This issue tracker is only for technical issues related to Bitcoin Core. -This issue tracker is only for technical issues related to bitcoin-core. +General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com. -General bitcoin questions and/or support requests and are best directed to the [Bitcoin StackExchange](https://bitcoin.stackexchange.com). +For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. -For reporting security issues, please read instructions at [https://bitcoincore.org/en/contact/](https://bitcoincore.org/en/contact/). +If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! --> -### Describe the issue +<!-- Describe the issue --> +<!--- What behavior did you expect? --> -### Can you reliably reproduce the issue? -#### If so, please list the steps to reproduce below: -1. -2. -3. +<!--- What was the actual behavior (provide screenshots if the issue is GUI-related)? --> -### Expected behaviour -Tell us what should happen +<!--- How reliably can you reproduce the issue, what are the steps to do so? --> -### Actual behaviour -Tell us what happens instead +<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? --> -### Screenshots. -If the issue is related to the GUI, screenshots can be added to this issue via drag & drop. +<!-- What type of machine are you observing the error on (OS/CPU and disk type)? --> -### What version of bitcoin-core are you using? -List the version number/commit ID, and if it is an official binary, self compiled or a distribution package such as PPA. - -### Machine specs: -- OS: -- CPU: -- RAM: -- Disk size: -- Disk Type (HD/SDD): - -### Any extra information that might be useful in the debugging process. -This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. +<!-- Any extra information that might be useful in the debugging process. --> +<!--- This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. --> diff --git a/.gitignore b/.gitignore index 60c26dae8b..ff297fbeca 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ src/qt/test/moc*.cpp Makefile bitcoin-qt Bitcoin-Qt.app +background.tiff* # Unit-tests Makefile.test diff --git a/.travis.yml b/.travis.yml index 0de7ca6f75..ab002acc53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required dist: trusty os: linux -language: generic +language: minimal cache: directories: - depends/built @@ -18,11 +18,10 @@ env: - CCACHE_COMPRESS=1 - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all matrix: # ARM - - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf python3-pip" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Win32 - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" # Qt4 & system libs @@ -40,16 +39,20 @@ env: before_install: - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/pyenv/d' | tr "\n" ":" | sed "s|::|:|g") install: - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi + - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then travis_retry pip3 install flake8 --user; fi before_script: - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/crypto/ctaes; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/secp256k1; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/univalue; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/leveldb; fi - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-rpc-mappings.py .; fi - - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/lint-all.sh; fi + - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/lint-all.sh; fi - unset CC; unset CXX - mkdir -p depends/SDKs depends/sdk-sources - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi @@ -76,7 +79,7 @@ script: - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning,dbcrash"; fi - - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --coverage --quiet ${extended}; fi + - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet ${extended}; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aed6d79542..8381bd2448 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,9 +24,9 @@ facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: - - Fork repository - - Create topic branch - - Commit patches + 1. Fork repository + 1. Create topic branch + 1. Commit patches The project coding conventions in the [developer notes](doc/developer-notes.md) must be adhered to. @@ -42,8 +42,8 @@ in init.cpp") in which case a single title line is sufficient. Commit messages s helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). -If a particular commit references another issue, please add the reference, for -example `refs #1234`, or `fixes #4321`. Using the `fixes` or `closes` keywords +If a particular commit references another issue, please add the reference. For +example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords will cause the corresponding issue to be closed when the pull request is merged. Please refer to the [Git manual](https://git-scm.com/doc) for more information @@ -85,7 +85,7 @@ Note that translations should not be submitted as pull requests, please see [Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) for more information on helping with translations. -If a pull request is specifically not to be considered for merging (yet) please +If a pull request is not to be considered for merging (yet), please prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) in the body of the pull request to indicate tasks are pending. @@ -186,7 +186,7 @@ In general, all pull requests must: the project (for example refactoring for modularisation); - Be well peer reviewed; - Have unit tests and functional tests where appropriate; - - Follow code style guidelines; + - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md)); - Not break the existing test suite; - Where bugs are fixed, where possible, there should be unit tests demonstrating the bug and also proving the fix. This helps prevent regression. diff --git a/Makefile.am b/Makefile.am index 3b62a10603..b24daf9905 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,7 +9,6 @@ SUBDIRS += doc/man endif .PHONY: deploy FORCE -GZIP_ENV="-9n" export PYTHONPATH if BUILD_BITCOIN_LIBS @@ -43,7 +42,11 @@ DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ $(top_srcdir)/contrib/init \ + $(top_srcdir)/contrib/install_db4.sh \ $(top_srcdir)/contrib/rpm +DIST_SHARE = \ + $(top_srcdir)/share/genbuild.sh \ + $(top_srcdir)/share/rpcauth BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py @@ -213,7 +216,7 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) EXTRA_DIST += \ test/util/bitcoin-util-test.py \ @@ -276,4 +279,4 @@ DISTCHECK_CONFIGURE_FLAGS = --enable-man clean-local: rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) - rm -rf test/functional/__pycache__ + rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache diff --git a/configure.ac b/configure.ac index 81c84a8af4..f7de4c9f9a 100644 --- a/configure.ac +++ b/configure.ac @@ -162,7 +162,7 @@ AC_ARG_ENABLE([ccache], AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], [enable lcov testing (default is no)])], - [use_lcov=yes], + [use_lcov=$enableval], [use_lcov=no]) AC_ARG_ENABLE([lcov-branch-coverage], @@ -241,6 +241,7 @@ if test "x$enable_werror" = "xyes"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -249,6 +250,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wformat],[CXXFLAGS="$CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[CXXFLAGS="$CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[CXXFLAGS="$CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[CXXFLAGS="$CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and @@ -657,6 +659,28 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +TEMP_LDFLAGS="$LDFLAGS" +LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" +AC_MSG_CHECKING([for thread_local support]) +AC_LINK_IFELSE([AC_LANG_SOURCE([ + #include <thread> + static thread_local int foo = 0; + static void run_thread() { foo++;} + int main(){ + for(int i = 0; i < 10; i++) { std::thread(run_thread).detach();} + return foo; + } + ])], + [ + AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) + AC_MSG_RESULT(yes) + ], + [ + AC_MSG_RESULT(no) + ] +) +LDFLAGS="$TEMP_LDFLAGS" + # Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> @@ -737,6 +761,9 @@ define(MINIMUM_REQUIRED_BOOST, 1.47.0) dnl Check for boost libs AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) +if test x$want_boost = xno; then + AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) +fi AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM AX_BOOST_PROGRAM_OPTIONS diff --git a/contrib/debian/bitcoin-qt.desktop b/contrib/debian/bitcoin-qt.desktop index 593d7584ab..204cdf99d0 100644 --- a/contrib/debian/bitcoin-qt.desktop +++ b/contrib/debian/bitcoin-qt.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Encoding=UTF-8 +Version=1.0 Name=Bitcoin Core Comment=Connect to the Bitcoin P2P Network Comment[de]=Verbinde mit dem Bitcoin peer-to-peer Netzwerk @@ -11,3 +11,4 @@ Type=Application Icon=bitcoin128 MimeType=x-scheme-handler/bitcoin; Categories=Office;Finance; +StartupWMClass=Bitcoin-qt diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 72d64ce62d..8e715bd9d8 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -15,6 +15,14 @@ Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> 2011, Matt Corallo <matt@bluematt.me> License: GPL-2+ +Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 +Copyright: 2008 Don Anderson <dda@sleepycat.com> +License: GNU-All-permissive-License + +Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 +Copyright: 2008 Paolo Bonzini <bonzini@gnu.org> +License: GNU-All-permissive-License + Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png @@ -106,6 +114,12 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +License: GNU-All-permissive-License + Copying and distribution of this file, with or without modification, are + permitted in any medium without royalty provided the copyright notice + and this notice are preserved. This file is offered as-is, without any + warranty. + License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the diff --git a/contrib/debian/examples/bitcoin.conf b/contrib/debian/examples/bitcoin.conf index 14a59fdf6b..4dd73162a2 100644 --- a/contrib/debian/examples/bitcoin.conf +++ b/contrib/debian/examples/bitcoin.conf @@ -77,9 +77,9 @@ #rpcpassword=YourSuperGreatPasswordNumber_DO_NOT_USE_THIS_OR_YOU_WILL_GET_ROBBED_385593 # # The second method `rpcauth` can be added to server startup argument. It is set at initialization time -# using the output from the script in share/rpcuser/rpcuser.py after providing a username: +# using the output from the script in share/rpcauth/rpcauth.py after providing a username: # -# ./share/rpcuser/rpcuser.py alice +# ./share/rpcauth/rpcauth.py alice # String to be appended to bitcoin.conf: # rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae # Your password: diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 967717e1e0..925d6a6252 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} SRCDIR=${SRCDIR:-$TOPDIR/src} diff --git a/contrib/devtools/git-subtree-check.sh b/contrib/devtools/git-subtree-check.sh index 2384d66cad..184951715e 100755 --- a/contrib/devtools/git-subtree-check.sh +++ b/contrib/devtools/git-subtree-check.sh @@ -18,7 +18,7 @@ find_latest_squash() sub= git log --grep="^git-subtree-dir: $dir/*\$" \ --pretty=format:'START %H%n%s%n%n%b%nEND%n' "$COMMIT" | - while read a b junk; do + while read a b _; do case "$a" in START) sq="$b" ;; git-subtree-mainline:) main="$b" ;; @@ -41,21 +41,17 @@ find_latest_squash() done } +# find latest subtree update latest_squash="$(find_latest_squash "$DIR")" if [ -z "$latest_squash" ]; then echo "ERROR: $DIR is not a subtree" >&2 exit 2 fi - set $latest_squash old=$1 rev=$2 -if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "ERROR: subtree commit $rev unavailable. Fetch/update the subtree repository" >&2 - exit 2 -fi -tree_subtree=$(git show -s --format="%T" $rev) -echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" + +# get the tree in the current commit tree_actual=$(git ls-tree -d "$COMMIT" "$DIR" | head -n 1) if [ -z "$tree_actual" ]; then echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2 @@ -69,9 +65,30 @@ if [ "d$tree_actual_type" != "dtree" ]; then echo "FAIL: subtree directory $DIR is not a tree in $COMMIT" >&2 exit 1 fi + +# get the tree at the time of the last subtree update +tree_commit=$(git show -s --format="%T" $old) +echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)" + +# ... and compare the actual tree with it +if [ "$tree_actual_tree" != "$tree_commit" ]; then + git diff $tree_commit $tree_actual_tree >&2 + echo "FAIL: subtree directory was touched without subtree merge" >&2 + exit 1 +fi + +# get the tree in the subtree commit referred to +if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then + echo "subtree commit $rev unavailable: cannot compare" >&2 + exit +fi +tree_subtree=$(git show -s --format="%T" $rev) +echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" + +# ... and compare the actual tree with it if [ "$tree_actual_tree" != "$tree_subtree" ]; then - git diff-tree $tree_actual_tree $tree_subtree >&2 - echo "FAIL: subtree directory tree doesn't match subtree commit tree" >&2 + echo "FAIL: subtree update commit differs from upstream tree!" >&2 exit 1 fi + echo "GOOD" diff --git a/contrib/devtools/lint-python.sh b/contrib/devtools/lint-python.sh new file mode 100755 index 0000000000..9303fcc8ef --- /dev/null +++ b/contrib/devtools/lint-python.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for specified flake8 warnings in python files. + +# F401: module imported but unused +flake8 --ignore=B,C,E,F,I,N,W --select=F401 . diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh index 8fdec21b0e..631fba9089 100755 --- a/contrib/gitian-build.sh +++ b/contrib/gitian-build.sh @@ -6,7 +6,6 @@ sign=false verify=false build=false -setupenv=false # Systems to build linux=true @@ -106,7 +105,7 @@ while :; do fi shift else - echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)\n' + echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)' exit 1 fi ;; @@ -189,7 +188,7 @@ then fi # Get signer -if [[ -n"$1" ]] +if [[ -n "$1" ]] then SIGNER=$1 shift diff --git a/contrib/gitian-keys/sjors-key.pgp b/contrib/gitian-keys/sjors-key.pgp new file mode 100644 index 0000000000..2b5acc82aa --- /dev/null +++ b/contrib/gitian-keys/sjors-key.pgp @@ -0,0 +1,76 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFWSwMoBEADG31O8+ex+xpgzVKQgF4iVRE5uBPT0+GM6FnwqIIhXVKiBLQh8 +YDhhgk6joh+vsLrFzKZ9kXwoiHN8y/AiNCQ0xjAUdpznD5xvHAaGIAlT/sodRNT+ +869WgT9G1uiVp0P4ucEeilmhCn9o51LqkS3roXkj0ec52b1pslUl2WKdu1ZD+Bj4 +3/oVZm7mmjkDwl0RHJQmqlK0bunq0jlVlgH5sdQfmLbCZaq3LhVPf73zt5qHH+J6 +ZbU7A4cqm2eN5SyH+Nno+cq3+vXmvVI+x/jPe/dPDCXaGWf5fWI/Lbk/mMP7JAl1 +6X44CN+hZHUnNuzeZt2/ROWZ0s0JJcjQkSe9noUQedjBAHX82s886vsFzOHvDtul +EuV/XAjUlkhMbhZkZaIq9ucqHmUBI4+OcFEIbbKc9TrKtJe+CYuWTNlomVk/iFr8 +zSm/S64NiqKi/BeQGgcsDZIaJDYfDP83esOOaaxFswHnJNtHnU1PwntrJtXft0dK +ydtlQZ6r96SYxLDTeGfC2SNk0zbnKAGvjj04vzQeN+JSRZ75tNKmgdbJdNL8wvPh +879TpCwMhNDvSRG+YqCe6whaJV76a+Doxg48HCJYaj6bnRn41/QGJEyL31I8l/7S +YsLLmAEbqwG7erYi7WZS3cRrGJI8RwohGMZf7yraqoaOgMKmtE/Sq0tLtwARAQAB +tCNTam9ycyBQcm92b29zdCA8c2pvcnNAc3Byb3Zvb3N0Lm5sPokCQAQTAQoAKgIb +AwUJB4YfgAULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCWWXcoAIZAQAKCRBX/5vb +zDAQCeYJD/47XDMfEMg4g4spo7k92XsNkvjlAhWvvxd+kxow/V8c64WQXody32FZ +HRSmK8dVjf9mIJMKkX4lpKpim7cQxsdTcorcdu+yk4TK+Wah61vsMhbSSllfHs1U ++q8jYMGnXTD+CY0aeTMrTfJcR2yN98jmNSWIL1qWmJ51RSTL6BQKb6eYtR7pWRkW +uMR6oFC09Db4fiKa4zhH81+/t0g+6pMY391gSluaS+OfNqGORCo+/IdG5IDzh5Vp +f19qXjd5oMsZQf6/P4b4XUktgl8RVRcNzdYGoXpcd8LpeHtEOh5I93ODmCwqd67b +YDlhDNN7iGhPndPEF6P4CNO/rXLPCZyMhRyt1dflu0KPCr+0AgR31cdhH/p7eCyj +FTE9gUgUHOG9OHdRoVXrwHYXwAiDBr2pp2giLpBsAwa4d2hXNDJ6wfMMCSOXKQlS +lHq06y/v/049DammkqW0XnEsU4qvsdteZ0jQu7Ob3LyGoytBIj8fn1OioT21W7wc +ns3/Tt4cQsn2ICBYB4PzqwkvGUp7fDwwHYw7rq6kvCEVDUDWMtVgQ8kjsh2OoU75 +eeteM1Q1fV06Wfn2Qct9bn0NKRGrA8mm3lrCWYCeGqJeBvC6kna1QgV53vYRLJod +w3Ql4+M9tUIi9uiGLvVaGZWO9wU1EwL+EAO+6D85h6QiJN7H8gcwUokCPQQTAQoA +JwUCVZLAygIbAwUJB4YfgAULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRBX/5vb +zDAQCauuD/9IDWhf/fTseA1Rt5i4gwK+8dCQjTlRS2cZtGc2aMX8w5XruDWnna1P +Mj/aVUncDrprRx9rxgEqIDyPheuJ6r7v6D8GjrpAjcG/BPNFtPaxQccbZbAYdzoj +Rrs+ttVIqS+wO7qLmQkKA4oGRMmgYh3VX8EBZNcvxaGCcJx0PfoqS8cPXTnCRHcg +Wx6kaFyuWtrTX+kCpDraB1KGtxedR4rzuOtUOLoqFOOfsQuOxPlKNNr9Zjc8x2o4 +5TtwbuoEog8FIEttY6NOywpsSsvYvNB4gq1fxO49H0pQopmJlOMatMH6IRT7BJJZ +cOoHOh4X/zItOJZtuCOT4u+Y2XOuyLcW83X5ymIR3ZCxedsLzjyiCWm61/znJVON +Ws8I+gShbvauahBCB9rOHqwM0QioJMc36hUPB21KghQS8RJpGwmtk1WhFFMtAsSJ +w+wRfy2d6u+lSGdlA+2hEyKVm/DNQMDCQVFx3lQ6YBwAwkSiLMylrPKvs56fUjRr +74qoPyDxuRMC+q+TThHsy5O9r31G+Dc3+H5k4iTk354Jshjltx/k2O732e9Vxyar +/U5P7UZqHHuJKXDihUFrcJZq+gk8sGEWzGG/wocce7ezrTnHqR8YA04BTA4PXQqZ +4N42f422YYGIH/3Nm6drQkbigekLw6wx+NrxtTsYg4eCtSsaUd/RjLQhU2pvcnMg +UHJvdm9vc3QgPHNqb3JzQGZyZWVkb20ubmw+iQI9BBMBCgAnBQJZZdyfAhsDBQkH +hh+ABQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEFf/m9vMMBAJEsIQAK4ihgRB +05QqETpWNeV/XSGBHQINuwwEDz/k8dAJ5Uo6OoSpDULa16fs/EgAV46wTSxfWuci +n2Fc1AWLeLDWOax/NlycL00VDHEwT2PCjcc5uMuwR4RUTciKyByT1u7BFToZ6PyL +mbU6u6whcQejl6Ci2kw0Mu4n4bKTS7OL4/w/EbdfMSpRi8wWmTPMB/aMjtS2Mxi/ +N+yQhJ9pReHADeCBoAjq0cUy+QbzvBwDCK4XWRzF7kiFuA7UW2r7/dX6l31mPfi/ +GLA5+ftPxJ6EH8cxToF70OWiSfhOTleaqZaHUOG0V7wV2lr/bwAYzpVlxeZSCIta +lAA9ZLzUD2hiHYcei6kc/YjIhmlml7O0FK1eBk7+bt5wr0nvWt4Lbha4y5LxBX8C +d7InvB3xUYHz+S5Ul4vp0Rzx97MBL4oX2ltBEDpc1CcOgzv4dcWMG9bbh9/SaI/G +RehAzwkbpVUl9AEUNKO0dNlZUdu8CkehHdPdz5sJyS/9zE0A7yIECDFP9Nrht0nK +MahBijm4K+jOiLOZ2xyfOX1pVWLqIXGQHKjfcD3oI3qvGrQYtxB5Dffb9ACFMpZO +z3jM8h2UAa2/KqA4MZiZG9N6uWHKkIAMMuXWs1s439WePvbQ+5aw/qPUAMyqA3XZ +dkfn8QWaJPR4nRM+McYBYuS4fKK9HRJWQgcQuQINBFWSwMoBEACzmkabZ8oHWJUE +beU7rJF/TMbwV1IFtFxJ/QlY8rE4VnHekPMvkLi/gjx3WY5nmMe+d4JYoK/uPNdt +y5u0QYgH2MB/jebk4gYXCAHIPpU38h9UgHRb6qV8OaqHhmoXvKwyz+1QPzyJpmgg +oCUN+OAroNjl7zhunE7w7EEddFQftfPoGKEUnTjv84QOCuAb46JsYyiNAc3h6okq +74hY7PKCv8IRGclMPjemhBT2LEenn1t4yi7a8W/hjIe44PmQiqQEXR17keqcP/ls +EH9xSST1v/70ieiPqb6zbHGWzjQxqpFUJxRU6OluBCy5pHVd8wfFGYrrbTpoxaUC +jyA2SLr1oZZ9gaGprt6X7FC5gpE5LV9essq3O5wwvoPbyMe1F5uFaxIPhlt55oEu +rwVWecFJ8tSjniF/WSkTcILrOmiQZ4mylXfOP9Wk38seZReCs799KEfKFlXHk89a +Sj3ZvaJQxwVCnvsAsbVKmmHZ5wPt+G2KfhOkkv2A1I/UyeTT7aXvt2vxDqGuG0su +Eo6QknM/2Sr5Uv7BwBeSIQ6llH5ZnqKz34+HjriP8YPWzvsC959GXsxS01dCSvUM +92j5PvTZzf5dt1CWHMeufAY5XIH+nftkRniuScRhJ7xK3tJ7wngg7UvdeZwJWqmK +lJ7GI38V8HIMnd2x28yiGpj1ue6T+QARAQABiQIlBBgBCgAPBQJVksDKAhsMBQkH +hh+AAAoJEFf/m9vMMBAJjeIP/1UBCi6gSXzpGJBLD2u4PcZJjXBJAImZdf1aCqfS +YZBCaA65UrM3uaVa7h8MGAJc9kDjpqHurjDmG3YWf33KvHWYmReQvX43pZmfF12s +X7FZgcCfgZJKKj+ri6oHQonZzUMrecEcAJLLaQoD3Du3iZpETiyRLL7sJ1lZSaCJ +gYKnN4WV5GypvdFvb8vSUBST2h0D6AewGKMNh8ruRlkIxI+YSlywgYIH+O0qNKqW +wBlZc/5f+JZ3hu+cjx/+Zn+w+saIb6SgySg0UzN35b2WM2YzrfQep4ah3NIxuC7e +qzmfV6GnRtuUrBLVJ8qyjif1JSM9tZfinnmAB4/U5Qfc+YYViIXMTljmHWvbokas +tTBfVAw74yWnkv4ZuXf5SkTmGwEMJUOat0TSr085Ck5y394bRepdI1Y+1cdqpwMQ +QmkKyvcBlREQ7Xk1UnDDR3o/2ieVuGGHRp8jmoWBWGq4Cm43fYOlVe+PcaX0tDns +Tmmh2uwEU/TXe5qGil51OlSM7qhAMqhWUIYphSOcdvApNXuiWMfnTdjsNygE4HVh +Jq4efJ/nlx5N+PNAK2GpzeUJQGyxiVsXybq+h8UlvytBsdz1X6ZYzBv1yYwANThU +rMB1s4tMaEugX0aNByLcsxuS4ixd2qzwkYVz25Aeko/U1v2/j2cIRtrTNgja3BKE +N5Ug +=80Es +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/gitian-keys/willyko-key.pgp b/contrib/gitian-keys/willyko-key.pgp new file mode 100644 index 0000000000..a3b58ad994 --- /dev/null +++ b/contrib/gitian-keys/willyko-key.pgp @@ -0,0 +1,85 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFgs/RoBEADFxycJTUvwqzBZZ0aBZXbmr8Ppd3EPrgBRd47k7uwanf7UFmvY +Xt4gMEI+EdV0GuoQ0SeoAmQqc5Fxu3AQe2XFbiF+ZNNYT3+V/5GAzWsAH22ncQr0 +AuK95pPi+PZ+M2h669cq/RzFUXZDew0NobR2oBS5h6g3rgmmejVLRqnUpWkkSrqi +aNgD2GSn8g820wM6LpdxcjTqmMpHHT5owAbv0UP3IcdtpBaS5McoUXK+OAdKK/Zw +JQ0J1kx4vIyNwuPD3klziGQw8Izb/gFpWg8XaJmMhD5BxNuXJC58Bj9/sFTc0GDQ +VKMFpYpNi8a6hLPFb4hMjYF77awoz57HtyOOsS03KO/57QE1htx+2NeDm4XkZSBk ++wrU3zgbtmOBcfzEHS/HrROksYDi+Qw3HZL98nfDEWNfsDzfhMZ9wHdM3NsR2xk6 +oNtX0CprS1n2Xr2AY9X1oNgiZCJaSftU67j3lr+9gHOH61ktxt3cUCDodUFjkpKn +r1CQ2LB63AoUbwGMAeozdXZWzbXJAJbcH9G77zEi9rW0WA2yMSxTXHlpE9MS0UcE +BVkIMv2b9iQzlhiS8jh8AiKFO1PuT26Cw52N/lSPhA81zw79pZfSYwKFICGHYfvw +ozZeN9Q+PPl5tqi/3SExxlZKe8EmaveTrUfKMBS4lQO2gWe0bCFgLOIzIwARAQAB +tB1XaWxseSBLbyA8d2lsbHlrQHN5c2NvaW4ub3JnPokCOAQTAQIAIgUCWCz9GgIb +AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQjjqPMkfby7+0wA//cX7Tc3Nz +19ApwSbGfC8pJA/nSybcVivroJRftpzeOmYrVM084T9REvYwugl89djvxn6m96iQ +kqoUGWhBVBtDReVCL7z53G42lHjemaFcxBhIazKxO0qvcc/UXUVOs2OdUbzObDFL +dHO5xBVqEnW3sq+r4blsXR8U79B9IIri4+2hy4OoEjYv9DzBaaoaqU+F3mudXbmo +R+hsWc+mklV++TX/kuw6EWT8tusFjXrfqqKcKPRPhbn48OSGWsEPc7yELf7pYFR8 +uDU40faJqkvQ83h5WMTDAhLxd/918ZitqBhjSP+7Humf2YhSto7YmtEWlbeAW+qy +TcBYkK6SJh8Do3xZd/prFBKEu395n5VQKuLjXaOjqMc1oDHQyPJJjXSN5thLHvan +z7nNLt2QZO/kxXITDdbWlktVe/WSoive7TuY4dGuX4Si2z9wyhFYxtZDsqE0qmqN +jIDAZ7u8Qq/LGqpdjOmYr2fEwHe1yVIS+BtVGvtShkX+J+QPb8qBl1d7Ii5i5Afl +GJoLLIUFkPcIRTYPZpppGSuqfyWdNnaasbLH44lxJisSMMw+fxZabt2bykYN/ZXa +RP/ItDj81vklg+n6r4f/nZTF1r0UUy4LbSbBY15B4Xm0Tdvh1PMfj/w2q10l7bZB +XLi9Z/QPaW7TyzaBuLkVckbVFn2nYnXfzHG5Ag0EWCz9GgEQALCgTibFnw+Q3PEL +G5/peQcQqHxrPAB37HV39B1DedGhVUa6aGSUaLoNMyUjUX1HWN3mWFKTYVB4CH5Y +xjaXUwxdwCZgBNe4TDglKFPuc+frlSTZxDVE9/fjArmrUP6TPU447ujspyngGLa5 +et5Uig/LxIX/+Mm0ZiYJxb1rMJwK998U1Ev1aHxgNjwTI2ehcKu8CAGOyflzh6a2 +iTBUmLfnQMv5248P2d4P8WDiPq61CWTYTMCFqHqkYKy7h9BYIuMajw3KsgOUNfL2 +1e9Ue8yv5UchZ+GDlBjidIkoK1nd2uJ0kPJkafLGWbcliJfvXxKliZnSbz1Cd4A0 +HDKKCwiuwSLy2aYbs7IRtAOyDER4+fjBcqtf0QTIvoAdNZ9gL64DKVaB58vuSixj +K1i83XbTOt3q821HxxBrX9u6HP2E5kFdxT2KHDbisAWNP0rFnHVpjugehKFfZb6q +jbDt3nQL5uCQ8gTNCd4fsoSK6KhCDjamDXlKmaGlxqwOV4W8ZwihoeGt690h7NIH +h4eiSmMOej3or32lcDETEwrjA2PxvcFsikFc56hRkTaSyyBEH2xhkRrjXMqiQfH0 +j7iOY2PWpFEuu2HVzqe5dBXzn9sMIwxeNCxR/P+xHMqPUlgD1SXEYCNLvvzD6p0+ +kqSe7PiJoEIv351T3hnBhQ6rK0ChABEBAAGJAh8EGAECAAkFAlgs/RoCGwwACgkQ +jjqPMkfby7/mQA//YsAOdDBl0GscB1PBNXi8VMlI7yG9cqiGrYnZX7h4wUoGEbPI +jap/PixIsxBCf1BqBRDJdFyvzH9amLlcaVNdCyh6Yt1Pi8kassmz/kbIYgpbFkIL +ES9N24N7BZ94P77OQy5wic+B4WqJnVrtKr9JBalgBSOMqtccYCma5Ew00mqp+FXM +suDyBk2HXyl+u6/rRmqZ+BoU8iRpus9F80LFKGEsAgjLjKv68KmApzjunzsBotKk +g9AsBk4ygbp+nECAtsxpbLMo4hPr4qWm7G4mU5g4xOK2chpAPeqyf0857RWgsXaO +kjrUu/M5Hme2eIlXwBF14ac4QPnY1rlAIaulvXzmQnMYQFZiw9vaTOdqBFHjkh7T +XYRAr589Woo25PfMJCbC+Rop6ku6sCFMorbBwojyRhFJnk9xsy5kP5D9IhkPAKu/ +/ABlei0xPOl/gCUUJP7aIikZgS5lAk1TSe/R+yV6ExNwudtLw1G0K2/sY3B4Xo3X +Q1lTAQPlnAIeK/vlbttLZNIBWquw4cPAkPpIyjmE1dd6jGQdUyZE22uPBx+gpq1w +AacmVLwvPMe1De0ilJOzj6KpXWBCwt0DWXWztovpBVcAC+qbTrZF9H5dllpqyzKt +OvxzGssjrX4rDkOx7MyVa2tnXmeCuSN/RvlOUwPvf5zYM8Wh9g7fc6jcDQu5Ag0E +WfkOfAEQAMNkzAQqSenpXtHsnuCqM1oMMF2kRzny/Jqh3q3BxZ8MHLDhoRRaTENu +lA4APRXMNM/wlZJUSLX8wWBhufnsPtMf6OOVMZ4AVbXHjUgyJ7lO1zHdj0u6PpYP +9gmHthIz7FF+cxHj4ziC4CmtRctrn+/U4MwYtNPhxkTnS26oOZes/HXMYSvQBMgT +AP27GNOBiJRthjIEITvSvS0YZOxgLtWgGiks/pGUw5wm1rguuQVyZ1/LfXBooYJo +u/v21AEjpuTg7JlwbqXr2k5LojAGq7AxDyWy21IW0E45Gog38zg/hwNll+hjRbSu +pipf74WXR6xMMlW6A+XWUvElkicfDx8e9LJUnqWbZ+FL7X4SB54ZHNCvfo/8Ug1V +2tiY9WbUZL9n5ZQHNlk3J+UK/KDvwey1VzKPFjpQNlfahhnppDGiCey+mERjI+75 +gPbk0ctOAEYXgLJjoonGX+iByAfY0YyJF281CtaK/sXQU+TzLLT15WET+gYGsJdY +xh1PdPscNdSgYudvbKZoFnqUwEGEfD8dT5bjOphfY5+LvGUR2GuLNZpMidcduTYf +SWAY/vQHQIJArXu29BKscm3tg6tzXu3l9p/bGIQUQB7obN91y3xD3BLICIPRGhKE +924wxxCuH1vLKmxWDdAAxKo+rEdLJ/rbZnjWQENEFiJ114fBk2NVABEBAAGJBEQE +GAEIAA8FAln5DnwCGwIFCQPCZwACKQkQjjqPMkfby7/BXSAEGQEIAAYFAln5DnwA +CgkQYFGSo/6YSmANoQ//SbcKxkop2zA2HrWS4THcEJQwSJ0KGAN/VB83JQhoWThX +CWxsFNJjBy7+rsoXd3wQG1/aN42nTuj+eh+R6WJJaqqnMqd52l4Kc1kJA6z4DGsy +3azCDvyzibM0AkJyMJyYi6HRKjzA4M+xKR1HoT/NdQUP5CBUVfvMblSaOWiw4rja +IhWcbgbQ+Zam/VaV5l1O90eaD9tL3twSfPLYZ/wkeO63jJKHBpI8fpMql/bLg9WD +Au3h/lU63NWe5lZO1z/jIdfiTSvg8nu162vcOgmUCWo9spkybjJd0Mx6ZId79rVo +58lwZ4QoaMgPGoVP67LyLOxJTIXeyG5xr1LxhMPMGbnBhlnMQrboLV9kPEL3raHE +EEKDTtZimVK3ZxmfyBd6MDmwcL/K73xu/R8be9TgdwD8/BZJSOTkO87qZ82G9T7E +oY5IHU+qd41/Yjbut7AVtAlCr5Lor31EYvZh3gI/H8uZFddOu37Ij7e9Fw2ywv3A +wPks89tfOvahkfCOJ29znB+uQYpJ461jjhdkB7EHG4ae07M5rRtkNbIc3dqbnMhz +VA3JpRJN77xPXV7uITHo1s+b50RvWmfYW91zvipaSZxbMLuGBMhn/1QaM1djLOYN +JordDBwEr2bi5a063yUbZrk6ddECuyxndDHWDNr+Tqx6o7lmAT48UJ199zA4scbf +2g/9EiRPGcRovsn1tUdjzfmWDxhrRV6F3rYJB1+i6Mqeg2iHHYxxiNDXcuWYXHQ/ +WPWLk5+lgh0rQbrE7InzEejoM0FIHzLTm0lSQpau50/PT2FiH6sOEEDyT2IhBtXX +eOnKAi1IfGNMzEaPEY8PXH78dEGv0iXIgy4l8Bc57q09Z9R/OUi+Yb1p+S5F/aOi +7Jd53GGE1bfBIlsMos092XoiMdvKmAczyCUIempKHUBPoqfJge77qk7zJKkyM3Dk +VX0lXLdhj0PfslFrNf2uRF4uZkmfUV7peeD023c0/SVp3ILUAVds52yawi6Exv4a +bbvhIw72fc31frCRBqc9HVsBraoozzE9bksG1MdNI3GgKxecOu9lldedlIqi4lO4 +7kTVDLEmcsQO+sSxkXQz2sMSD01CQndpPuhFNlqvVnfK+Kv8pSG37VzSSQz1nt5K +w/fJBo4T/ztR7D9RzbSDxBP8Jjaa+UYabjab5HcE0JI4CpgmzIOB7qPVbYCn+LNX +c8Xw5/9iTw+ayawl7PCGRfd14/OPRzI8vS0I9bF8AG84XM46yxAtYieH/9RI3b6/ +GiQYDkBNi6Kb1LfSzx8oKAkbMgiy4y3vWxLQnE34bAoXjGiYdAMliOsyGcvmnObD +GmSTIlIqunq60CyhaUSIkl2VRhjzz0igfS9751XEvnjeXDc= +=PVBi +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/init/bitcoind.openrc b/contrib/init/bitcoind.openrc index 50377c0995..86222295db 100644 --- a/contrib/init/bitcoind.openrc +++ b/contrib/init/bitcoind.openrc @@ -1,4 +1,4 @@ -#!/sbin/runscript +#!/sbin/openrc-run # backward compatibility for existing gentoo layout # diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 9132957c38..ee113d7615 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -1,22 +1,25 @@ +# It is not recommended to modify this file in-place, because it will +# be overwritten during package upgrades. If you want to add further +# options or overwrite existing ones then use +# $ systemctl edit bitcoind.service +# See "man systemd.service" for details. + +# Note that almost all daemon options could be specified in +# /etc/bitcoin/bitcoin.conf + [Unit] -Description=Bitcoin's distributed currency daemon +Description=Bitcoin daemon After=network.target [Service] +ExecStart=/usr/bin/bitcoind -daemon -conf=/etc/bitcoin/bitcoin.conf -pid=/run/bitcoind/bitcoind.pid +# Creates /run/bitcoind owned by bitcoin +RuntimeDirectory=bitcoind User=bitcoin -Group=bitcoin - Type=forking -PIDFile=/var/lib/bitcoind/bitcoind.pid -ExecStart=/usr/bin/bitcoind -daemon -pid=/var/lib/bitcoind/bitcoind.pid \ --conf=/etc/bitcoin/bitcoin.conf -datadir=/var/lib/bitcoind -disablewallet - -Restart=always +PIDFile=/run/bitcoind/bitcoind.pid +Restart=on-failure PrivateTmp=true -TimeoutStopSec=60s -TimeoutStartSec=2s -StartLimitInterval=120s -StartLimitBurst=5 [Install] WantedBy=multi-user.target diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh new file mode 100755 index 0000000000..b57ade139d --- /dev/null +++ b/contrib/install_db4.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# Install libdb4.8 (Berkeley DB). + +set -e + +if [ -z "${1}" ]; then + echo "Usage: ./install_db4.sh <base-dir> [<extra-bdb-configure-flag> ...]" + echo + echo "Must specify a single argument: the directory in which db4 will be built." + echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." + exit 1 +fi + +expand_path() { + echo "$(cd "${1}" && pwd -P)" +} + +BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_VERSION='db-4.8.30.NC' +BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' +BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" + +check_exists() { + which "$1" >/dev/null 2>&1 +} + +sha256_check() { + # Args: <sha256_hash> <filename> + # + if check_exists sha256sum; then + echo "${1} ${2}" | sha256sum -c + elif check_exists sha256; then + if [ "$(uname)" = "FreeBSD" ]; then + sha256 -c "${1}" "${2}" + else + echo "${1} ${2}" | sha256 -c + fi + else + echo "${1} ${2}" | shasum -a 256 -c + fi +} + +http_get() { + # Args: <url> <filename> <sha256_hash> + # + # It's acceptable that we don't require SSL here because we manually verify + # content hashes below. + # + if [ -f "${2}" ]; then + echo "File ${2} already exists; not downloading again" + elif check_exists curl; then + curl --insecure "${1}" -o "${2}" + else + wget --no-check-certificate "${1}" -O "${2}" + fi + + sha256_check "${3}" "${2}" +} + +mkdir -p "${BDB_PREFIX}" +http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" +tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" +cd "${BDB_PREFIX}/${BDB_VERSION}/" + +# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) +CLANG_CXX11_PATCH_URL='https://gist.githubusercontent.com/LnL7/5153b251fd525fe15de69b67e63a6075/raw/7778e9364679093a32dec2908656738e16b6bdcb/clang.patch' +CLANG_CXX11_PATCH_HASH='7a9a47b03fd5fb93a16ef42235fa9512db9b0829cfc3bdf90edd3ec1f44d637c' +http_get "${CLANG_CXX11_PATCH_URL}" clang.patch "${CLANG_CXX11_PATCH_HASH}" +patch -p2 < clang.patch + +cd build_unix/ + +"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ + --enable-cxx --disable-shared --with-pic --prefix="${BDB_PREFIX}" \ + "${@}" + +make install + +echo +echo "db4 build complete." +echo +echo 'When compiling bitcoind, run `./configure` in the following way:' +echo +echo " export BDB_PREFIX='${BDB_PREFIX}'" +echo ' ./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" ...' diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 7f017bb4f1..3379a4599c 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -40,7 +40,7 @@ grep CodeResources < "${TEMPLIST}" | while read i; do RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" DIRNAME="`dirname "${RESOURCE}"`" mkdir -p "${DIRNAME}" - echo "Adding resource for: "${TARGETFILE}"" + echo "Adding resource for: \"${TARGETFILE}\"" cp "${i}" "${RESOURCE}" done diff --git a/contrib/tidy_datadir.sh b/contrib/tidy_datadir.sh index 8960f8811d..b845b34e41 100755 --- a/contrib/tidy_datadir.sh +++ b/contrib/tidy_datadir.sh @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. if [ -d "$1" ]; then - cd "$1" + cd "$1" || exit 1 else echo "Usage: $0 <datadir>" >&2 echo "Removes obsolete Bitcoin database files" >&2 diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp new file mode 100644 index 0000000000..0f6d993fd2 --- /dev/null +++ b/contrib/valgrind.supp @@ -0,0 +1,43 @@ +# Valgrind suppressions file for Bitcoin. +# +# Includes known Valgrind warnings in our dependencies that cannot be fixed +# in-tree. +# +# Example use: +# $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +{ + Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + obj:*/libstdc++.* + fun:call_init.part.0 + fun:call_init + fun:_dl_init + obj:*/ld-*.so +} +{ + Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 + Memcheck:Cond + obj:*/libdb_cxx-*.so + fun:__log_put + obj:*/libdb_cxx-*.so + fun:__log_put_record +} +{ + Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7leveldbL10InitModuleEv +} +{ + Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldbL14InitDefaultEnvEv +} diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index b01e2a6d39..8f3e4b8063 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -9,7 +9,7 @@ REVSIG=false IFS=' ' if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then - GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" else # Note how we've disabled SHA1 with the --weak-digest option, disabling # signatures - including selfsigs - that use SHA1. While you might think that @@ -24,7 +24,7 @@ else case "$LINE" in "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr - GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" ;; # We assume if you're running 2.1+, you're probably running 2.1.10+ # gpg will fail otherwise @@ -32,7 +32,7 @@ else # gpg will fail otherwise esac done - [ "$GPG_RES" = "" ] && GPG_RES="$(echo "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" + [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" fi for LINE in $(echo "$GPG_RES"); do case "$LINE" in @@ -46,14 +46,19 @@ for LINE in $(echo "$GPG_RES"); do REVSIG=true GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" ;; + "[GNUPG:] EXPKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" + ;; esac done if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else - echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null fi diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index 74b7f38375..532b97a438 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -12,8 +12,6 @@ VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit") REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") -HAVE_FAILED=false - HAVE_GNU_SHA512=1 [ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0 @@ -35,11 +33,12 @@ fi NO_SHA1=1 PREV_COMMIT="" +INITIAL_COMMIT="${CURRENT_COMMIT}" while true; do if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then - echo "There is a valid path from "$CURRENT_COMMIT" to $VERIFIED_ROOT where all commits are signed!" - exit 0; + echo "There is a valid path from \"$INITIAL_COMMIT\" to $VERIFIED_ROOT where all commits are signed!" + exit 0 fi if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then @@ -95,9 +94,9 @@ while true; do FILE_HASHES="" for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do if [ "$HAVE_GNU_SHA512" = 1 ]; then - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } ) + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST _; echo $FIRST; } ) else - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST OTHER; echo $FIRST; } ) + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST _; echo $FIRST; } ) fi [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"' ' diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index 409f517c9f..e0266bf08a 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -33,7 +33,7 @@ if [ ! -d "$WORKINGDIR" ]; then mkdir "$WORKINGDIR" fi -cd "$WORKINGDIR" +cd "$WORKINGDIR" || exit 1 #test if a version number has been passed as an argument if [ -n "$1" ]; then @@ -76,8 +76,6 @@ if [ -n "$1" ]; then BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/" fi fi - - SIGNATUREFILE="$BASEDIR$SIGNATUREFILENAME" else echo "Error: need to specify a version on the command line" exit 2 @@ -89,7 +87,7 @@ WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1) #and then see if wget completed successfully if [ $? -ne 0 ]; then echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" - echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: "$VERSIONPREFIX"0.10.4-"$RCVERSIONSTRING"1)" + echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" echo "wget output:" echo "$WGETOUT"|sed 's/^/\t/g' exit 2 diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 5cc19761d4..5efd3d2187 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -8,8 +8,8 @@ Bitcoin should be started with the command line arguments: bitcoind -testnet -daemon \ - -zmqpubhashblock=tcp://127.0.0.1:28332 \ -zmqpubrawtx=tcp://127.0.0.1:28332 \ + -zmqpubrawblock=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashblock=tcp://127.0.0.1:28332 diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py index bfb7ea9eae..7032d475e8 100755 --- a/contrib/zmq/zmq_sub3.4.py +++ b/contrib/zmq/zmq_sub3.4.py @@ -8,8 +8,8 @@ Bitcoin should be started with the command line arguments: bitcoind -testnet -daemon \ - -zmqpubhashblock=tcp://127.0.0.1:28332 \ -zmqpubrawtx=tcp://127.0.0.1:28332 \ + -zmqpubrawblock=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashblock=tcp://127.0.0.1:28332 diff --git a/depends/README.md b/depends/README.md index 6053c531b4..99eef1952c 100644 --- a/depends/README.md +++ b/depends/README.md @@ -28,6 +28,22 @@ Common `host-platform-triplets` for cross compilation are: No other options are needed, the paths are automatically configured. +Install the required dependencies: Ubuntu & Debian +-------------------------------------------------- + +For macOS cross compilation: + + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools + +For Win32/Win64 cross compilation: + +- see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) + +For linux (including i386, ARM) cross compilation: + + sudo apt-get install curl g++-aarch64-linux-gnu g++-4.8-aarch64-linux-gnu gcc-4.8-aarch64-linux-gnu binutils-aarch64-linux-gnu g++-arm-linux-gnueabihf g++-4.8-arm-linux-gnueabihf gcc-4.8-arm-linux-gnueabihf binutils-arm-linux-gnueabihf g++-4.8-multilib gcc-4.8-multilib binutils-gold bsdmainutils + + Dependency Options: The following can be set when running make: make FOO=bar diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 01146c26f6..3f97221e10 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,19 +1,18 @@ package=zeromq -$(package)_version=4.1.5 -$(package)_download_path=https://github.com/zeromq/zeromq4-1/releases/download/v$($(package)_version)/ +$(package)_version=4.2.2 +$(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=04aac57f081ffa3a2ee5ed04887be9e205df3a7ddade0027460b8042432bdbcf -$(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b100cd38acecc16ce7db02905e27c.patch +$(package)_sha256_hash=5b23f4ca9ef545d5bd3af55d305765e3ee06b986263b31967435d285a3e6df6b +$(package)_patches=0001-fix-build-with-older-mingw64.patch define $(package)_set_vars - $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve + $(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf $(package)_config_opts_linux=--with-pic $(package)_cxxflags=-std=c++11 endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/9114d3957725acd34aa8b8d011585812f3369411.patch && \ - patch -p1 < $($(package)_patch_dir)/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch && \ + patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ ./autogen.sh endef @@ -22,7 +21,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) libzmq.la + $(MAKE) src/libzmq.la endef define $(package)_stage_cmds @@ -30,5 +29,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ rm -rf bin share endef diff --git a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch new file mode 100644 index 0000000000..a6c508fb8a --- /dev/null +++ b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch @@ -0,0 +1,30 @@ +From 1a159c128c69a42d90819375c06a39994f3fbfc1 Mon Sep 17 00:00:00 2001 +From: Cory Fields <cory-nospam-@coryfields.com> +Date: Tue, 28 Nov 2017 20:33:25 -0500 +Subject: [PATCH] fix build with older mingw64 + +--- + src/windows.hpp | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/windows.hpp b/src/windows.hpp +index 99e889d..e69038e 100644 +--- a/src/windows.hpp ++++ b/src/windows.hpp +@@ -55,6 +55,13 @@ + #include <winsock2.h> + #include <windows.h> + #include <mswsock.h> ++ ++#if defined __MINGW64_VERSION_MAJOR && __MINGW64_VERSION_MAJOR < 4 ++// Workaround for mingw-w64 < v4.0 which did not include ws2ipdef.h in iphlpapi.h. ++// Fixed in mingw-w64 by 9bd8fe9148924840d315b4c915dd099955ea89d1. ++#include <ws2def.h> ++#include <ws2ipdef.h> ++#endif + #include <iphlpapi.h> + + #if !defined __MINGW32__ +-- +2.7.4 + diff --git a/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch b/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch deleted file mode 100644 index f704b3d94f..0000000000 --- a/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9114d3957725acd34aa8b8d011585812f3369411 Mon Sep 17 00:00:00 2001 -From: Jeroen Ooms <jeroenooms@gmail.com> -Date: Tue, 20 Oct 2015 13:10:38 +0200 -Subject: [PATCH] enable static libraries on mingw - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 393505b..e92131a 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -265,7 +265,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- AC_MSG_ERROR([Building static libraries is not supported under MinGW32]) -+ CPPFLAGS="-DZMQ_STATIC" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch b/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch deleted file mode 100644 index 9aff2c179a..0000000000 --- a/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9e6745c12e0b100cd38acecc16ce7db02905e27c Mon Sep 17 00:00:00 2001 -From: David Millard <dmillard10@gmail.com> -Date: Tue, 10 May 2016 13:53:53 -0700 -Subject: [PATCH] Fix autotools for static MinGW builds - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 5a0fa14..def6ea7 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -259,7 +259,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- CPPFLAGS="-DZMQ_STATIC" -+ CPPFLAGS="-DZMQ_STATIC $CPPFLAGS" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/doc/bips.md b/doc/bips.md index bc8dcb6fb3..fbff94a329 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -33,3 +33,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**): * [`BIP 145`](https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki): getblocktemplate updates for Segregated Witness as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). +* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): NODE_NETWORK_LIMITED service bit [signaling only] is supported as of **v0.16.0** ([PR 10740](https://github.com/bitcoin/bitcoin/pull/10740)). diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 4ea8e30b53..cd1d217b47 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,10 +1,10 @@ OpenBSD build guide ====================== -(updated for OpenBSD 6.1) +(updated for OpenBSD 6.2) This guide describes how to build bitcoind and command-line utilities on OpenBSD. -As OpenBSD is most common as a server OS, we will not bother with the GUI. +OpenBSD is most commonly used as a server OS, so this guide does not contain instructions for building the GUI. Preparation ------------- @@ -12,10 +12,13 @@ Preparation Run the following as root to install the base dependencies for building: ```bash -pkg_add gmake libtool libevent +pkg_add git gmake libevent libtool pkg_add autoconf # (select highest version, e.g. 2.69) pkg_add automake # (select highest version, e.g. 1.15) -pkg_add python # (select highest version, e.g. 3.5) +pkg_add python # (select highest version, e.g. 3.6) +pkg_add boost + +git clone https://github.com/bitcoin/bitcoin.git ``` See [dependencies.md](dependencies.md) for a complete overview. @@ -23,83 +26,37 @@ See [dependencies.md](dependencies.md) for a complete overview. GCC ------- -The default C++ compiler that comes with OpenBSD 5.9 is g++ 4.2. This version is old (from 2007), and is not able to compile the current version of Bitcoin Core, primarily as it has no C++11 support, but even before there were issues. So here we will be installing a newer compiler: +The default C++ compiler that comes with OpenBSD 6.2 is g++ 4.2.1. This version is old (from 2007), and is not able to compile the current version of Bitcoin Core because it has no C++11 support. We'll install a newer version of GCC: ```bash -pkg_add g++ # (select newest 4.x version, e.g. 4.9.3) -``` - -This compiler will not overwrite the system compiler, it will be installed as `egcc` and `eg++` in `/usr/local/bin`. - -### Building boost - -Do not use `pkg_add boost`! The boost version installed thus is compiled using the `g++` compiler not `eg++`, which will result in a conflict between `/usr/local/lib/libestdc++.so.XX.0` and `/usr/lib/libstdc++.so.XX.0`, resulting in a test crash: - - test_bitcoin:/usr/lib/libstdc++.so.57.0: /usr/local/lib/libestdc++.so.17.0 : WARNING: symbol(_ZN11__gnu_debug17_S_debug_me ssagesE) size mismatch, relink your program - ... - Segmentation fault (core dumped) + pkg_add g++ + ``` -This makes it necessary to build boost, or at least the parts used by Bitcoin Core, manually: - -``` -# Pick some path to install boost to, here we create a directory within the bitcoin directory -BITCOIN_ROOT=$(pwd) -BOOST_PREFIX="${BITCOIN_ROOT}/boost" -mkdir -p $BOOST_PREFIX - -# Fetch the source and verify that it is not tampered with -curl -o boost_1_64_0.tar.bz2 https://netcologne.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 -echo '7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 boost_1_64_0.tar.bz2' | sha256 -c -# MUST output: (SHA256) boost_1_64_0.tar.bz2: OK -tar -xjf boost_1_64_0.tar.bz2 - -# Boost 1.64 needs one small patch for OpenBSD -cd boost_1_64_0 -# Also here: https://gist.githubusercontent.com/laanwj/bf359281dc319b8ff2e1/raw/92250de8404b97bb99d72ab898f4a8cb35ae1ea3/patch-boost_test_impl_execution_monitor_ipp.patch -patch -p0 < /usr/ports/devel/boost/patches/patch-boost_test_impl_execution_monitor_ipp - -# Build w/ minimum configuration necessary for bitcoin -echo 'using gcc : : eg++ : <cxxflags>"-fvisibility=hidden -fPIC" <linkflags>"" <archiver>"ar" <striper>"strip" <ranlib>"ranlib" <rc>"" : ;' > user-config.jam -config_opts="runtime-link=shared threadapi=pthread threading=multi link=static variant=release --layout=tagged --build-type=complete --user-config=user-config.jam -sNO_BZIP2=1" -./bootstrap.sh --without-icu --with-libraries=chrono,filesystem,program_options,system,thread,test -./b2 -d2 -j2 -d1 ${config_opts} --prefix=${BOOST_PREFIX} stage -./b2 -d0 -j4 ${config_opts} --prefix=${BOOST_PREFIX} install -``` + This compiler will not overwrite the system compiler, it will be installed as `egcc` and `eg++` in `/usr/local/bin`. ### Building BerkeleyDB BerkeleyDB is only necessary for the wallet functionality. To skip this, pass `--disable-wallet` to `./configure`. -See "Berkeley DB" in [build_unix.md](build_unix.md) for instructions on how to build BerkeleyDB 4.8. -You cannot use the BerkeleyDB library from ports, for the same reason as boost above (g++/libstd++ incompatibility). +It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library +from ports, for the same reason as boost above (g++/libstd++ incompatibility). +If you have to build it yourself, you can use [the installation script included +in contrib/](contrib/install_db4.sh) like so -```bash -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BITCOIN_ROOT=$(pwd) -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -curl -o db-4.8.30.NC.tar.gz 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256 -c -# MUST output: (SHA256) db-4.8.30.NC.tar.gz: OK -tar -xzf db-4.8.30.NC.tar.gz - -# Build the library and install to specified prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX CC=egcc CXX=eg++ CPP=ecpp -make install # do NOT use -jX, this is broken +```shell +./contrib/install_db4.sh `pwd` CC=egcc CXX=eg++ CPP=ecpp ``` +from the root of the repository. + ### Resource limits The standard ulimit restrictions in OpenBSD are very strict: data(kbytes) 1572864 -This is, unfortunately, no longer enough to compile some `.cpp` files in the project, -at least with gcc 4.9.3 (see issue https://github.com/bitcoin/bitcoin/issues/6658). +This, unfortunately, may no longer be enough to compile some `.cpp` files in the project, +at least with GCC 4.9.4 (see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). If your user is in the `staff` group the limit can be raised with: ulimit -d 3000000 @@ -118,59 +75,32 @@ export AUTOCONF_VERSION=2.69 # replace this with the autoconf version that you i export AUTOMAKE_VERSION=1.15 # replace this with the automake version that you installed ./autogen.sh ``` -Make sure `BDB_PREFIX` and `BOOST_PREFIX` are set to the appropriate paths from the above steps. +Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. To configure with wallet: ```bash -./configure --with-gui=no --with-boost=$BOOST_PREFIX \ - CC=egcc CXX=eg++ CPP=ecpp \ +./configure --with-gui=no CC=egcc CXX=eg++ CPP=ecpp \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --with-boost=$BOOST_PREFIX \ - CC=egcc CXX=eg++ CPP=ecpp +./configure --disable-wallet --with-gui=no CC=egcc CXX=eg++ CPP=ecpp ``` Build and run the tests: ```bash -gmake # can use -jX here for parallelism +gmake # use -jX here for parallelism gmake check ``` -Clang (not currently working) +Clang ------------------------------ -WARNING: This is outdated, needs to be updated for OpenBSD 6.0 and re-tried. - -Using a newer g++ results in linking the new code to a new libstdc++. -Libraries built with the old g++, will still import the old library. -This gives conflicts, necessitating rebuild of all C++ dependencies of the application. - -With clang this can - at least theoretically - be avoided because it uses the -base system's libstdc++. - ```bash -pkg_add llvm boost -``` +pkg_add llvm -```bash ./configure --disable-wallet --with-gui=no CC=clang CXX=clang++ -gmake +gmake # use -jX here for parallelism +gmake check ``` - -However, this does not appear to work. Compilation succeeds, but link fails -with many 'local symbol discarded' errors: - - local symbol 150: discarded in section `.text._ZN10tinyformat6detail14FormatIterator6finishEv' from libbitcoin_util.a(libbitcoin_util_a-random.o) - local symbol 151: discarded in section `.text._ZN10tinyformat6detail14FormatIterator21streamStateFromFormatERSoRjPKcii' from libbitcoin_util.a(libbitcoin_util_a-random.o) - local symbol 152: discarded in section `.text._ZN10tinyformat6detail12convertToIntIA13_cLb0EE6invokeERA13_Kc' from libbitcoin_util.a(libbitcoin_util_a-random.o) - -According to similar reported errors this is a binutils (ld) issue in 2.15, the -version installed by OpenBSD 5.7: - -- http://openbsd-archive.7691.n7.nabble.com/UPDATE-cppcheck-1-65-td248900.html -- https://llvm.org/bugs/show_bug.cgi?id=9758 - -There is no known workaround for this. diff --git a/doc/build-osx.md b/doc/build-osx.md index 836c856a64..cbc5288c16 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -26,6 +26,20 @@ If you want to build the disk image with `make deploy` (.dmg / optional), you ne NOTE: Building with Qt4 is still supported, however, could result in a broken UI. Building with Qt5 is recommended. +Berkeley DB +----------- +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](/contrib/install_db4.sh) +like so + +```shell +./contrib/install_db4.sh . +``` + +from the root of the repository. + +**Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). + Build Bitcoin Core ------------------------ @@ -96,6 +110,6 @@ Uncheck everything except Qt Creator during the installation process. Notes ----- -* Tested on OS X 10.8 through 10.12 on 64-bit Intel processors only. +* Tested on OS X 10.8 through 10.13 on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) diff --git a/doc/build-unix.md b/doc/build-unix.md index 8a102abaea..af567cadeb 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -165,33 +165,16 @@ turned off by default. See the configure options for upnp behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself: +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](/contrib/install_db4.sh) +like so -```bash -BITCOIN_ROOT=$(pwd) - -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -wget 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256sum -c -# -> db-4.8.30.NC.tar.gz: OK -tar -xzvf db-4.8.30.NC.tar.gz - -# Build the library and install to our prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX -make install - -# Configure Bitcoin Core to use our own-built instance of BDB -cd $BITCOIN_ROOT -./autogen.sh -./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" # (other args...) +```shell +./contrib/install_db4.sh `pwd` ``` +from the root of the repository. + **Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). Boost @@ -329,17 +312,13 @@ You need to use GNU make (`gmake`) instead of `make`. For the wallet (optional): - pkg install db5 - -This will give a warning "configure: WARNING: Found Berkeley DB other -than 4.8; wallets opened by this build will not be portable!", but as FreeBSD never -had a binary release, this may not matter. If backwards compatibility -with 4.8-built Bitcoin Core is needed follow the steps under "Berkeley DB" above. + ./contrib/install_db4.sh `pwd` + setenv BDB_PREFIX $PWD/db4 Then build using: ./autogen.sh - ./configure --with-incompatible-bdb BDB_CFLAGS="-I/usr/local/include/db5" BDB_LIBS="-L/usr/local/lib -ldb_cxx-5" + ./configure BDB_CFLAGS="-I${BDB_PREFIX}/include" BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx" gmake *Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). diff --git a/doc/build-windows.md b/doc/build-windows.md index 0d96af26a2..9e0e66e522 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -3,61 +3,67 @@ WINDOWS BUILD NOTES Below are some notes on how to build Bitcoin Core for Windows. -Most developers use cross-compilation from Ubuntu to build executables for -Windows. Cross-compilation is also used to build the release binaries. +The options known to work for building Bitcoin Core on Windows are: -Currently only building on Ubuntu Trusty 14.04 or Ubuntu Zesty 17.04 or later is supported. -Building on Ubuntu Xenial 16.04 is known to be broken, see extensive discussion in issue [8732](https://github.com/bitcoin/bitcoin/issues/8732). -While it may be possible to do so with work arounds, it's potentially dangerous and not recommended. +* On Linux using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Trusty 14.04 is recommended +and is the platform used to build the Bitcoin Core Windows release binaries. +* On Windows using [Windows +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about) and the Mingw-w64 cross compiler tool chain. -While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64), -using the Windows Subsystem For Linux is the most straightforward. If you are building with -another method, please contribute the instructions here for others who are running versions -of Windows that are not compatible with the Windows Subsystem for Linux. +Other options which may work but which have not been extensively tested are (please contribute instructions): -Compiling with Windows Subsystem For Linux -------------------------------------------- +* On Windows using a POSIX compatibility layer application such as [cygwin](http://www.cygwin.com/) or [msys2](http://www.msys2.org/). +* On Windows using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). + +Installing Windows Subsystem for Linux +--------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. +the need for a separate Linux VM or server. Note that while WSL can be installed with +other Linux variants, such as OpenSUSE, the following instructions have only been +tested with Ubuntu. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of Windows](https://msdn.microsoft.com/en-us/commandline/wsl/install_guide). -To get the bash shell, you must first activate the feature in Windows. +Full instructions to install WSL are available on the above link. +To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: -1. Turn on Developer Mode - * Open Settings -> Update and Security -> For developers - * Select the Developer Mode radio button - * Restart if necessary -2. Enable the Windows Subsystem for Linux feature +1. Enable the Windows Subsystem for Linux feature * From Start, search for "Turn Windows features on or off" (type 'turn') - * Select Windows Subsystem for Linux (beta) + * Select Windows Subsystem for Linux * Click OK * Restart if necessary +2. Install Ubuntu + * Open Microsoft Store and search for Ubuntu or use [this link](https://www.microsoft.com/store/productId/9NBLGGH4MSV6) + * Click Install 3. Complete Installation - * Open a cmd prompt and type "bash" - * Accept the license + * Open a cmd prompt and type "Ubuntu" * Create a new UNIX user account (this is a separate account from your Windows account) After the bash shell is active, you can follow the instructions below, starting with the "Cross-compilation" section. Compiling the 64-bit version is recommended but it is possible to compile the 32-bit version. -Cross-compilation -------------------- +Cross-compilation for Ubuntu and Windows Subsystem for Linux +------------------------------------------------------------ + +At the time of writing the Windows Subsystem for Linux installs Ubuntu Xenial 16.04. The Mingw-w64 package +for Ubuntu Xenial does not produce working executables for some of the Bitcoin Core applications. +It is possible to build on Ubuntu Xenial by installing the cross compiler packages from Ubuntu Zesty, see the steps below. +Building on Ubuntu Zesty 17.04 up to 17.10 has been verified to work. -These steps can be performed on, for example, an Ubuntu VM. The depends system +The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. First, install the general dependencies: - sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl + sudo apt install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git A host toolchain (`build-essential`) is necessary because some dependency packages (such as `protobuf`) need to build host utilities that are used in the @@ -65,20 +71,44 @@ build process. See also: [dependencies.md](dependencies.md). -If you're building on Ubuntu 17.04 or later, run these two commands, selecting the 'posix' variant for both, -to work around issues with mingw-w64. See issue [8732](https://github.com/bitcoin/bitcoin/issues/8732) for more information. -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ -sudo update-alternatives --config x86_64-w64-mingw32-gcc -``` - ## Building for 64-bit Windows -To build executables for Windows 64-bit, install the following dependencies: +The first step is to install the mingw-w64 cross-compilation tool chain. Due to different Ubuntu +packages for each distribution and problems with the Xenial packages the steps for each are different. - sudo apt-get install g++-mingw-w64-x86-64 mingw-w64-x86-64-dev +Common steps to install mingw32 cross compiler tool chain: -Then build using: + sudo apt install g++-mingw-w64-x86-64 + +Ubuntu Trusty 14.04: + + No further steps required + +Ubuntu Xenial 16.04 and Windows Subsystem for Linux <sup>[1](#footnote1),[2](#footnote2)</sup>: + + sudo apt install software-properties-common + sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu zesty universe" + sudo apt update + sudo apt upgrade + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Ubuntu Zesty 17.04 <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Once the tool chain is installed the build steps are common: + +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin + +Once the source code is ready the build steps are below. PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -92,7 +122,21 @@ Then build using: To build executables for Windows 32-bit, install the following dependencies: - sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev + sudo apt install g++-mingw-w64-i686 mingw-w64-i686-dev + +For Ubuntu Xenial 16.04, Ubuntu Zesty 17.04 and Windows Subsystem for Linux <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config i686-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin Then build using: @@ -117,3 +161,20 @@ as they appear in the release `.zip` archive. This can be done in the following way. This will install to `c:\workspace\bitcoin`, for example: make install DESTDIR=/mnt/c/workspace/bitcoin + +Footnotes +--------- + +<a name="footnote1">1</a>: There is currently a bug in the 64 bit Mingw-w64 cross compiler packaged for WSL/Ubuntu Xenial 16.04 that +causes two of the bitcoin executables to crash shortly after start up. The bug is related to the +-fstack-protector-all g++ compiler flag which is used to mitigate buffer overflows. +Installing the Mingw-w64 packages from the Ubuntu 17 distribution solves the issue, however, this is not +an officially supported approach and it's only recommended if you are prepared to reinstall WSL/Ubuntu should +something break. + +<a name="footnote2">2</a>: Starting from Ubuntu Xenial 16.04 both the 32 and 64 bit Mingw-w64 packages install two different +compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more +efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers +required to support win32 threads conflict with some of the classes in the C++11 standard library in particular std::mutex. +It's not possible to build the bitcoin code using the win32 version of the Mingw-w64 cross compilers (at least not without +modifying headers in the bitcoin source code). diff --git a/doc/dependencies.md b/doc/dependencies.md index 964c03ba88..5c5645de97 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Expat | [2.2.1](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](http://download.savannah.gnu.org/releases/freetype) | | No | | | -| GCC | | [4.7+](https://gcc.gnu.org/) | | | | +| GCC | | [4.8+](https://gcc.gnu.org/) | | | | | HarfBuzz-NG | | | | | | | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L75) | @@ -27,5 +27,5 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Qt | [5.7.1](https://download.qt.io/official_releases/qt/) | 4.7+ | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L94) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L93) (Linux only) | -| ZeroMQ | [4.1.5](https://github.com/zeromq/libzmq/releases) | | No | | | +| ZeroMQ | [4.2.2](https://github.com/zeromq/libzmq/releases) | | No | | | | zlib | [1.2.11](http://zlib.net/) | | | | No | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 33c6ab9cb3..9dc63a1e4b 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -167,6 +167,37 @@ can be very difficult to track down. Compiling with -DDEBUG_LOCKORDER (configure CXXFLAGS="-DDEBUG_LOCKORDER -g") inserts run-time checks to keep track of which locks are held, and adds warnings to the debug.log file if inconsistencies are detected. +**Valgrind suppressions file** + +Valgrind is a programming tool for memory debugging, memory leak detection, and +profiling. The repo contains a Valgrind suppressions file +([`valgrind.supp`](https://github.com/bitcoin/bitcoin/blob/master/contrib/valgrind.supp)) +which includes known Valgrind warnings in our dependencies that cannot be fixed +in-tree. Example use: + +```shell +$ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +$ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ + --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +$ valgrind -v --leak-check=full src/bitcoind -printtoconsole +``` + +**compiling for test coverage** + +LCOV can be used to generate a test coverage report based upon `make check` +execution. LCOV must be installed on your system (e.g. the `lcov` package +on Debian/Ubuntu). + +To enable LCOV report generation during test runs: + +```shell +./configure --enable-lcov +make +make cov + +# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +``` + Locking/mutex usage notes ------------------------- @@ -461,6 +492,14 @@ namespace { - *Rationale*: Avoids confusion about the namespace context +- Prefer `#include <primitives/transaction.h>` bracket syntax instead of + `#include "primitives/transactions.h"` quote syntax when possible. + + - *Rationale*: Bracket syntax is less ambiguous because the preprocessor + searches a fixed list of include directories without taking location of the + source file into account. This allows quoted includes to stand out more when + the location of the source file actually is relevant. + GUI ----- @@ -644,3 +683,16 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if new data in the response is needed. + +- Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with + `getblockchaininfo`'s state immediately prior to the call's execution. Wallet + RPCs whose behavior does *not* depend on the current chainstate may omit this + call. + + - *Rationale*: In previous versions of Bitcoin Core, the wallet was always + in-sync with the chainstate (by virtue of them all being updated in the + same cs_main lock). In order to maintain the behavior that wallet RPCs + return results as of at least the highest best-known block an RPC + client may be aware of prior to entering a wallet RPC call, we must block + until the wallet is caught up to the chainstate as of the RPC call's entry. + This also makes the API much easier for RPC clients to reason about. diff --git a/doc/files.md b/doc/files.md index 3d603445fb..2eac7ed664 100644 --- a/doc/files.md +++ b/doc/files.md @@ -6,13 +6,16 @@ * blocks/rev000??.dat; block undo data (custom); since 0.8.0 (format changed since pre-0.8) * blocks/index/*; block index (LevelDB); since 0.8.0 * chainstate/*; block chain state database (LevelDB); since 0.8.0 -* database/*: BDB database environment; only used for wallet since 0.8.0 -* db.log: wallet database log file +* database/*: BDB database environment; only used for wallet since 0.8.0; moved to wallets/ directory on new installs since 0.16.0 +* db.log: wallet database log file; moved to wallets/ directory on new installs since 0.16.0 * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 * mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 -* wallet.dat: personal wallet (BDB) with keys and transactions +* wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.16.0 +* wallets/database/*: BDB database environment; used for wallets since 0.16.0 +* wallets/db.log: wallet database log file; since 0.16.0 +* wallets/wallet.dat: personal wallet (BDB) with keys and transactions; since 0.16.0 * .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown): since 0.12.0 * onion_private_key: cached Tor hidden service private key for `-listenonion`: since 0.12.0 * guisettings.ini.bak: backup of former GUI settings after `-resetguisettings` is used diff --git a/doc/release-notes.md b/doc/release-notes.md index 4ecca7897c..d92666da72 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -20,7 +20,7 @@ How to Upgrade ============== If you are running an older version, shut it down. Wait until it has completely -shut down (which might take a few minutes for older versions), then run the +shut down (which might take a few minutes for older versions), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). @@ -56,17 +56,10 @@ frequently tested on them. Notable changes =============== -Miner block size limiting deprecated ------------------------------------- - -Though blockmaxweight has been preferred for limiting the size of blocks returned by -getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished -to limit their block size directly. Using this option resulted in a few UI issues as -well as non-optimal fee selection and ever-so-slightly worse performance, and has thus -now been deprecated. Further, the blockmaxsize option is now used only to calculate an -implied blockmaxweight, instead of limiting block size directly. Any miners who wish -to limit their blocks by size, instead of by weight, will have to do so manually by -removing transactions from their block template directly. +GCC 4.8.x +-------------- +The minimum version of GCC required to compile Bitcoin Core is now 4.8. No effort will be +made to support older versions of GCC. See discussion in issue #11732 for more information. HD-wallets by default --------------------- @@ -74,17 +67,50 @@ Due to a backward-incompatible change in the wallet database, wallets created with version 0.16.0 will be rejected by previous versions. Also, version 0.16.0 will only create hierarchical deterministic (HD) wallets. +Replace-By-Fee by default in GUI +-------------------------------- +The send screen now uses BIP-125 RBF by default, regardless of `-walletrbf`. +There is a checkbox to mark the transaction as final. + +The RPC default remains unchanged: to use RBF, launch with `-walletrbf=1` or +use the `replaceable` argument for individual transactions. + +Custom wallet directories +--------------------- +The ability to specify a directory other than the default data directory in which to store +wallets has been added. An existing directory can be specified using the `-walletdir=<dir>` +argument. Wallets loaded via `-wallet` arguments must be in this wallet directory. Care should be taken +when choosing a wallet directory location, as if it becomes unavailable during operation, +funds may be lost. + +Default wallet directory change +-------------------------- +On new installations (if the data directory doesn't exist), wallets will now be stored in a +new `wallets/` subdirectory inside the data directory. If this `wallets/` subdirectory +doesn't exist (i.e. on existing nodes), the current datadir root is used instead, as it was. + Low-level RPC changes ---------------------- -- The "currentblocksize" value in getmininginfo has been removed. - The deprecated RPC `getinfo` was removed. It is recommended that the more specific RPCs are used: * `getblockchaininfo` * `getnetworkinfo` * `getwalletinfo` * `getmininginfo` +- The wallet RPC `getreceivedbyaddress` will return an error if called with an address not in the wallet. + +Changed command-line options +----------------------------- +- `-debuglogfile=<file>` can be used to specify an alternative debug logging file. + +Renamed script for creating JSON-RPC credentials +----------------------------- +The `share/rpcuser/rpcuser.py` script was renamed to `share/rpcauth/rpcauth.py`. This script can be +used to create `rpcauth` credentials for a JSON-RPC user. + -- `dumpwallet` no longer allows overwriting files. This is a security measure - as well as prevents dangerous user mistakes. +- `dumpwallet` now includes hex-encoded scripts from the wallet in the dumpfile, and + `importwallet` now imports these scripts, but corresponding addresses may not be added + correctly or a manual rescan may be required to find relevant transactions. Credits ======= diff --git a/doc/release-notes/release-notes-0.15.1.md b/doc/release-notes/release-notes-0.15.1.md new file mode 100644 index 0000000000..75d2e09714 --- /dev/null +++ b/doc/release-notes/release-notes-0.15.1.md @@ -0,0 +1,277 @@ +Bitcoin Core version *0.15.1* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.15.1/> + +or + + <https://bitcoin.org/bin/bitcoin-core-0.15.1/> + +This is a new minor version release, including various bugfixes and +performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +The first time you run version 0.15.0 or higher, your chainstate database will +be converted to a new format, which will take anywhere from a few minutes to +half an hour, depending on the speed of your machine. + +The file format of `fee_estimates.dat` changed in version 0.15.0. Hence, a +downgrade from version 0.15 or upgrade to version 0.15 will cause all fee +estimates to be discarded. + +Note that the block database format also changed in version 0.8.0 and there is no +automatic upgrade code from before version 0.8 to version 0.15.0. Upgrading +directly from 0.7.x and earlier without redownloading the blockchain is not supported. +However, as usual, old wallet versions are still supported. + +Downgrading warning +------------------- + +The chainstate database for this release is not compatible with previous +releases, so if you run 0.15 and then decide to switch back to any +older version, you will need to run the old release with the `-reindex-chainstate` +option to rebuild the chainstate data structures in the old format. + +If your node has pruning enabled, this will entail re-downloading and +processing the entire blockchain. + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. Windows XP is not supported. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + + +Notable changes +=============== + +Network fork safety enhancements +-------------------------------- + +A number of changes to the way Bitcoin Core deals with peer connections and invalid blocks +have been made, as a safety precaution against blockchain forks and misbehaving peers. + +- Unrequested blocks with less work than the minimum-chain-work are now no longer processed even +if they have more work than the tip (a potential issue during IBD where the tip may have low-work). +This prevents peers wasting the resources of a node. + +- Peers which provide a chain with less work than the minimum-chain-work during IBD will now be disconnected. + +- For a given outbound peer, we now check whether their best known block has at least as much work as our tip. If it +doesn't, and if we still haven't heard about a block with sufficient work after a 20 minute timeout, then we send +a single getheaders message, and wait 2 more minutes. If after two minutes their best known block has insufficient +work, we disconnect that peer. We protect 4 of our outbound peers from being disconnected by this logic to prevent +excessive network topology changes as a result of this algorithm, while still ensuring that we have a reasonable +number of nodes not known to be on bogus chains. + +- Outbound (non-manual) peers that serve us block headers that are already known to be invalid (other than compact +block announcements, because BIP 152 explicitly permits nodes to relay compact blocks before fully validating them) +will now be disconnected. + +- If the chain tip has not been advanced for over 30 minutes, we now assume the tip may be stale and will try to connect +to an additional outbound peer. A periodic check ensures that if this extra peer connection is in use, we will disconnect +the peer that least recently announced a new block. + +- The set of all known invalid-themselves blocks (i.e. blocks which we attempted to connect but which were found to be +invalid) are now tracked and used to check if new headers build on an invalid chain. This ensures that everything that +descends from an invalid block is marked as such. + + +Miner block size limiting deprecated +------------------------------------ + +Though blockmaxweight has been preferred for limiting the size of blocks returned by +getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished +to limit their block size directly. Using this option resulted in a few UI issues as +well as non-optimal fee selection and ever-so-slightly worse performance, and has thus +now been deprecated. Further, the blockmaxsize option is now used only to calculate an +implied blockmaxweight, instead of limiting block size directly. Any miners who wish +to limit their blocks by size, instead of by weight, will have to do so manually by +removing transactions from their block template directly. + + +GUI settings backed up on reset +------------------------------- + +The GUI settings will now be written to `guisettings.ini.bak` in the data directory before wiping them when +the `-resetguisettings` argument is used. This can be used to retroactively troubleshoot issues due to the +GUI settings. + + +Duplicate wallets disallowed +---------------------------- + +Previously, it was possible to open the same wallet twice by manually copying the wallet file, causing +issues when both were opened simultaneously. It is no longer possible to open copies of the same wallet. + + +Debug `-minimumchainwork` argument added +---------------------------------------- + +A hidden debug argument `-minimumchainwork` has been added to allow a custom minimum work value to be used +when validating a chain. + + +Low-level RPC changes +---------------------- + +- The "currentblocksize" value in getmininginfo has been removed. + +- `dumpwallet` no longer allows overwriting files. This is a security measure + as well as prevents dangerous user mistakes. + +- `backupwallet` will now fail when attempting to backup to source file, rather than + destroying the wallet. + +- `listsinceblock` will now throw an error if an unknown `blockhash` argument + value is passed, instead of returning a list of all wallet transactions since + the genesis block. The behaviour is unchanged when an empty string is provided. + +0.15.1 Change log +================= + +### Mining +- #11100 `7871a7d` Fix confusing blockmax{size,weight} options, dont default to throwing away money (TheBlueMatt) + +### RPC and other APIs +- #10859 `2a5d099` gettxout: Slightly improve doc and tests (jtimon) +- #11267 `b1a6c94` update cli for estimate\*fee argument rename (laanwj) +- #11483 `20cdc2b` Fix importmulti bug when importing an already imported key (pedrobranco) +- #9937 `a43be5b` Prevent `dumpwallet` from overwriting files (laanwj) +- #11465 `405e069` Update named args documentation for importprivkey (dusty-wil) +- #11131 `b278a43` Write authcookie atomically (laanwj) +- #11565 `7d4546f` Make listsinceblock refuse unknown block hash (ryanofsky) +- #11593 `8195cb0` Work-around an upstream libevent bug (theuni) + +### P2P protocol and network code +- #11397 `27e861a` Improve and document SOCKS code (laanwj) +- #11252 `0fe2a9a` When clearing addrman clear mapInfo and mapAddr (instagibbs) +- #11527 `a2bd86a` Remove my testnet DNS seed (schildbach) +- #10756 `0a5477c` net processing: swap out signals for an interface class (theuni) +- #11531 `55b7abf` Check that new headers are not a descendant of an invalid block (more effeciently) (TheBlueMatt) +- #11560 `49bf090` Connect to a new outbound peer if our tip is stale (sdaftuar) +- #11568 `fc966bb` Disconnect outbound peers on invalid chains (sdaftuar) +- #11578 `ec8dedf` Add missing lock in ProcessHeadersMessage(...) (practicalswift) +- #11456 `6f27965` Replace relevant services logic with a function suite (TheBlueMatt) +- #11490 `bf191a7` Disconnect from outbound peers with bad headers chains (sdaftuar) + +### Validation +- #10357 `da4908c` Allow setting nMinimumChainWork on command line (sdaftuar) +- #11458 `2df65ee` Don't process unrequested, low-work blocks (sdaftuar) + +### Build system +- #11440 `b6c0209` Fix validationinterface build on super old boost/clang (TheBlueMatt) +- #11530 `265bb21` Add share/rpcuser to dist. source code archive (MarcoFalke) + +### GUI +- #11334 `19d63e8` Remove custom fee radio group and remove nCustomFeeRadio setting (achow101) +- #11198 `7310f1f` Fix display of package name on 'open config file' tooltip (esotericnonsense) +- #11015 `6642558` Add delay before filtering transactions (lclc) +- #11338 `6a62c74` Backup former GUI settings on `-resetguisettings` (laanwj) +- #11335 `8d13b42` Replace save|restoreWindowGeometry with Qt functions (MeshCollider) +- #11237 `2e31b1d` Fixing division by zero in time remaining (MeshCollider) +- #11247 `47c02a8` Use IsMine to validate custom change address (MarcoFalke) + +### Wallet +- #11017 `9e8aae3` Close DB on error (kallewoof) +- #11225 `6b4d9f2` Update stored witness in AddToWallet (sdaftuar) +- #11126 `2cb720a` Acquire cs_main lock before cs_wallet during wallet initialization (ryanofsky) +- #11476 `9c8006d` Avoid opening copied wallet databases simultaneously (ryanofsky) +- #11492 `de7053f` Fix leak in CDB constructor (promag) +- #11376 `fd79ed6` Ensure backupwallet fails when attempting to backup to source file (tomasvdw) +- #11326 `d570aa4` Fix crash on shutdown with invalid wallet (MeshCollider) + +### Tests and QA +- #11399 `a825d4a` Fix bip68-sequence rpc test (jl2012) +- #11150 `847c75e` Add getmininginfo test (mess110) +- #11407 `806c78f` add functional test for mempoolreplacement command line arg (instagibbs) +- #11433 `e169349` Restore bitcoin-util-test py2 compatibility (MarcoFalke) +- #11308 `2e1ac70` zapwallettxes: Wait up to 3s for mempool reload (MarcoFalke) +- #10798 `716066d` test bitcoin-cli (jnewbery) +- #11443 `019c492` Allow "make cov" out-of-tree; Fix rpc mapping check (MarcoFalke) +- #11445 `51bad91` 0.15.1 Backports (MarcoFalke) +- #11319 `2f0b30a` Fix error introduced into p2p-segwit.py, and prevent future similar errors (sdaftuar) +- #10552 `e4605d9` Tests for zmqpubrawtx and zmqpubrawblock (achow101) +- #11067 `eeb24a3` TestNode: Add wait_until_stopped helper method (MarcoFalke) +- #11068 `5398f20` Move wait_until to util (MarcoFalke) +- #11125 `812c870` Add bitcoin-cli -stdin and -stdinrpcpass functional tests (promag) +- #11077 `1d80d1e` fix timeout issues from TestNode (jnewbery) +- #11078 `f1ced0d` Make p2p-leaktests.py more robust (jnewbery) +- #11210 `f3f7891` Stop test_bitcoin-qt touching ~/.bitcoin (MeshCollider) +- #11234 `f0b6795` Remove redundant testutil.cpp|h files (MeshCollider) +- #11215 `cef0319` fixups from set_test_params() (jnewbery) +- #11345 `f9cf7b5` Check connectivity before sending in assumevalid.py (jnewbery) +- #11091 `c276c1e` Increase initial RPC timeout to 60 seconds (laanwj) +- #10711 `fc2aa09` Introduce TestNode (jnewbery) +- #11230 `d8dd8e7` Fixup dbcrash interaction with add_nodes() (jnewbery) +- #11241 `4424176` Improve signmessages functional test (mess110) +- #11116 `2c4ff35` Unit tests for script/standard and IsMine functions (jimpo) +- #11422 `a36f332` Verify DBWrapper iterators are taking snapshots (TheBlueMatt) +- #11121 `bb5e7cb` TestNode tidyups (jnewbery) +- #11521 `ca0f3f7` travis: move back to the minimal image (theuni) +- #11538 `adbc9d1` Fix race condition failures in replace-by-fee.py, sendheaders.py (sdaftuar) +- #11472 `4108879` Make tmpdir option an absolute path, misc cleanup (MarcoFalke) +- #10853 `5b728c8` Fix RPC failure testing (again) (jnewbery) +- #11310 `b6468d3` Test listwallets RPC (mess110) + +### Miscellaneous +- #11377 `75997c3` Disallow uncompressed pubkeys in bitcoin-tx [multisig] output adds (TheBlueMatt) +- #11437 `dea3b87` [Docs] Update Windows build instructions for using WSL and Ubuntu 17.04 (fanquake) +- #11318 `8b61aee` Put back inadvertently removed copyright notices (gmaxwell) +- #11442 `cf18f42` [Docs] Update OpenBSD Build Instructions for OpenBSD 6.2 (fanquake) +- #10957 `50bd3f6` Avoid returning a BIP9Stats object with uninitialized values (practicalswift) +- #11539 `01223a0` [verify-commits] Allow revoked keys to expire (TheBlueMatt) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andreas Schildbach +- Andrew Chow +- Chris Moore +- Cory Fields +- Cristian Mircea Messel +- Daniel Edgecumbe +- Donal OConnor +- Dusty Williams +- fanquake +- Gregory Sanders +- Jim Posen +- John Newbery +- Johnson Lau +- João Barbosa +- Jorge Timón +- Karl-Johan Alm +- Lucas Betschart +- MarcoFalke +- Matt Corallo +- Paul Berg +- Pedro Branco +- Pieter Wuille +- practicalswift +- Russell Yanofsky +- Samuel Dobson +- Suhas Daftuar +- Tomas van der Wansem +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/share/genbuild.sh b/share/genbuild.sh index 32ef2a5755..419e0da0fd 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. if [ $# -gt 1 ]; then - cd "$2" + cd "$2" || exit 1 fi if [ $# -gt 0 ]; then FILE="$1" diff --git a/share/rpcauth/README.md b/share/rpcauth/README.md new file mode 100644 index 0000000000..389278a125 --- /dev/null +++ b/share/rpcauth/README.md @@ -0,0 +1,10 @@ +RPC Tools +--------------------- + +### [RPCAuth](/share/rpcauth) ### + +Create login credentials for a JSON-RPC user. + +Usage: + + ./rpcauth.py <username> diff --git a/share/rpcuser/rpcuser.py b/share/rpcauth/rpcauth.py index 63c69e308a..6d9b44f699 100755 --- a/share/rpcuser/rpcuser.py +++ b/share/rpcauth/rpcauth.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/share/rpcuser/README.md b/share/rpcuser/README.md deleted file mode 100644 index 12a8e6fb0c..0000000000 --- a/share/rpcuser/README.md +++ /dev/null @@ -1,10 +0,0 @@ -RPC Tools ---------------------- - -### [RPCUser](/share/rpcuser) ### - -Create an RPC user login credential. - -Usage: - - ./rpcuser.py <username> diff --git a/src/Makefile.am b/src/Makefile.am index 90deff48b0..4b65774fc6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -168,6 +168,7 @@ BITCOIN_CORE_H = \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ + wallet/walletutil.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -249,6 +250,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ + wallet/walletutil.cpp \ $(BITCOIN_CORE_H) # crypto primitives library @@ -500,10 +502,6 @@ clean-local: ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ -.mm.o: - $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ - $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) -c -o $@ $< - check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index e4b64c1ca7..0bdde06772 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -362,12 +362,12 @@ RES_MOVIES = $(wildcard $(srcdir)/qt/res/movies/spinner-*.png) BITCOIN_RC = qt/res/bitcoin-qt-res.rc -BITCOIN_QT_INCLUDES = -I$(builddir)/qt -I$(srcdir)/qt -I$(srcdir)/qt/forms \ - -I$(builddir)/qt/forms -DQT_NO_KEYWORDS +BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) +qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) @@ -438,7 +438,7 @@ $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ -CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno +CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc CLEANFILES += $(CLEAN_QT) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d3e7b5da12..49a814c3f6 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -80,10 +80,10 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ test/uint256_tests.cpp \ - test/univalue_tests.cpp \ test/util_tests.cpp if ENABLE_WALLET @@ -96,7 +96,7 @@ BITCOIN_TESTS += \ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) +test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 7f85c16585..7d7f34863d 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -3,17 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrdb.h" - -#include "addrman.h" -#include "chainparams.h" -#include "clientversion.h" -#include "fs.h" -#include "hash.h" -#include "random.h" -#include "streams.h" -#include "tinyformat.h" -#include "util.h" +#include <addrdb.h> + +#include <addrman.h> +#include <chainparams.h> +#include <clientversion.h> +#include <hash.h> +#include <random.h> +#include <streams.h> +#include <tinyformat.h> +#include <util.h> namespace { diff --git a/src/addrdb.h b/src/addrdb.h index d930de204d..098da4240e 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include "fs.h" -#include "serialize.h" +#include <fs.h> +#include <serialize.h> #include <string> #include <map> diff --git a/src/addrman.cpp b/src/addrman.cpp index a56bb4f9c1..372cac8483 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" +#include <addrman.h> -#include "hash.h" -#include "serialize.h" -#include "streams.h" +#include <hash.h> +#include <serialize.h> +#include <streams.h> int CAddrInfo::GetTriedBucket(const uint256& nKey) const { @@ -390,9 +390,9 @@ int CAddrMan::Check_() if (vRandom.size() != nTried + nNew) return -7; - for (std::map<int, CAddrInfo>::iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - int n = (*it).first; - CAddrInfo& info = (*it).second; + for (const auto& entry : mapInfo) { + int n = entry.first; + const CAddrInfo& info = entry.second; if (info.fInTried) { if (!info.nLastSuccess) return -1; diff --git a/src/addrman.h b/src/addrman.h index 18f3062287..ea289d508c 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -6,12 +6,12 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include "netaddress.h" -#include "protocol.h" -#include "random.h" -#include "sync.h" -#include "timedata.h" -#include "util.h" +#include <netaddress.h> +#include <protocol.h> +#include <random.h> +#include <sync.h> +#include <timedata.h> +#include <util.h> #include <map> #include <set> @@ -313,9 +313,9 @@ public: s << nUBuckets; std::map<int, int> mapUnkIds; int nIds = 0; - for (std::map<int, CAddrInfo>::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - mapUnkIds[(*it).first] = nIds; - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const CAddrInfo &info = entry.second; if (info.nRefCount) { assert(nIds != nNew); // this means nNew was wrong, oh ow s << info; @@ -323,8 +323,8 @@ public: } } nIds = 0; - for (std::map<int, CAddrInfo>::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + const CAddrInfo &info = entry.second; if (info.fInTried) { assert(nIds != nTried); // this means nTried was wrong, oh ow s << info; @@ -455,6 +455,7 @@ public: void Clear() { + LOCK(cs); std::vector<int>().swap(vRandom); nKey = GetRandHash(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index b4952af6f4..738224fb70 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "arith_uint256.h" +#include <arith_uint256.h> -#include "uint256.h" -#include "utilstrencodings.h" -#include "crypto/common.h" +#include <uint256.h> +#include <utilstrencodings.h> +#include <crypto/common.h> #include <stdio.h> #include <string.h> diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 5fd4fe96cf..1009fe1cc8 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -25,7 +25,7 @@ template<unsigned int BITS> class base_uint { protected: - enum { WIDTH=BITS/32 }; + static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: diff --git a/src/base58.cpp b/src/base58.cpp index 9d5a2f4964..cd087e7997 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" +#include <base58.h> -#include "bech32.h" -#include "hash.h" -#include "script/script.h" -#include "uint256.h" -#include "utilstrencodings.h" +#include <bech32.h> +#include <hash.h> +#include <script/script.h> +#include <uint256.h> +#include <utilstrencodings.h> #include <boost/variant/apply_visitor.hpp> #include <boost/variant/static_visitor.hpp> diff --git a/src/base58.h b/src/base58.h index 9dc4234248..fee79f02bf 100644 --- a/src/base58.h +++ b/src/base58.h @@ -14,11 +14,11 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include "chainparams.h" -#include "key.h" -#include "pubkey.h" -#include "script/standard.h" -#include "support/allocators/zeroafterfree.h" +#include <chainparams.h> +#include <key.h> +#include <pubkey.h> +#include <script/standard.h> +#include <support/allocators/zeroafterfree.h> #include <string> #include <vector> diff --git a/src/bech32.cpp b/src/bech32.cpp index 573eac58bb..274782e467 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bech32.h" +#include <bech32.h> namespace { diff --git a/src/bench/Examples.cpp b/src/bench/Examples.cpp index 314947d48c..536e450940 100644 --- a/src/bench/Examples.cpp +++ b/src/bench/Examples.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "validation.h" -#include "utiltime.h" +#include <bench/bench.h> +#include <validation.h> +#include <utiltime.h> // Sanity test: this should loop ten times, and // min/max/average should be close to 100ms. diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 65e27a615d..2d9a9f2908 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "validation.h" -#include "base58.h" +#include <validation.h> +#include <base58.h> #include <array> #include <vector> @@ -22,7 +22,7 @@ static void Base58Encode(benchmark::State& state) } }; while (state.KeepRunning()) { - EncodeBase58(buff.begin(), buff.end()); + EncodeBase58(buff.data(), buff.data() + buff.size()); } } diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 7b307d6f42..1482452814 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -2,35 +2,31 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "perf.h" +#include <bench/bench.h> +#include <bench/perf.h> #include <assert.h> #include <iostream> #include <iomanip> -#include <sys/time.h> benchmark::BenchRunner::BenchmarkMap &benchmark::BenchRunner::benchmarks() { static std::map<std::string, benchmark::BenchFunction> benchmarks_map; return benchmarks_map; } -static double gettimedouble(void) { - struct timeval tv; - gettimeofday(&tv, nullptr); - return tv.tv_usec * 0.000001 + tv.tv_sec; -} - benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) { benchmarks().insert(std::make_pair(name, func)); } void -benchmark::BenchRunner::RunAll(double elapsedTimeForOne) +benchmark::BenchRunner::RunAll(benchmark::duration elapsedTimeForOne) { perf_init(); - std::cout << "#Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "," + if (std::ratio_less_equal<benchmark::clock::period, std::micro>::value) { + std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n"; + } + std::cout << "#Benchmark" << "," << "count" << "," << "min(ns)" << "," << "max(ns)" << "," << "average(ns)" << "," << "min_cycles" << "," << "max_cycles" << "," << "average_cycles" << "\n"; for (const auto &p: benchmarks()) { @@ -46,16 +42,17 @@ bool benchmark::State::KeepRunning() ++count; return true; } - double now; + time_point now; + uint64_t nowCycles; if (count == 0) { - lastTime = beginTime = now = gettimedouble(); + lastTime = beginTime = now = clock::now(); lastCycles = beginCycles = nowCycles = perf_cpucycles(); } else { - now = gettimedouble(); - double elapsed = now - lastTime; - double elapsedOne = elapsed / (countMask + 1); + now = clock::now(); + auto elapsed = now - lastTime; + auto elapsedOne = elapsed / (countMask + 1); if (elapsedOne < minTime) minTime = elapsedOne; if (elapsedOne > maxTime) maxTime = elapsedOne; @@ -70,8 +67,8 @@ bool benchmark::State::KeepRunning() // The restart avoids including the overhead of this code in the measurement. countMask = ((countMask<<3)|7) & ((1LL<<60)-1); count = 0; - minTime = std::numeric_limits<double>::max(); - maxTime = std::numeric_limits<double>::min(); + minTime = duration::max(); + maxTime = duration::zero(); minCycles = std::numeric_limits<uint64_t>::max(); maxCycles = std::numeric_limits<uint64_t>::min(); return true; @@ -94,9 +91,13 @@ bool benchmark::State::KeepRunning() assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above"); // Output results - double average = (now-beginTime)/count; + // Duration casts are only necessary here because hardware with sub-nanosecond clocks + // will lose precision. + int64_t min_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(minTime).count(); + int64_t max_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(maxTime).count(); + int64_t avg_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>((now-beginTime)/count).count(); int64_t averageCycles = (nowCycles-beginCycles)/count; - std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << minTime << "," << maxTime << "," << average << "," + std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << min_elapsed << "," << max_elapsed << "," << avg_elapsed << "," << minCycles << "," << maxCycles << "," << averageCycles << "\n"; std::cout.copyfmt(std::ios(nullptr)); diff --git a/src/bench/bench.h b/src/bench/bench.h index 79109eaa56..071a5dc9c7 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -9,6 +9,7 @@ #include <limits> #include <map> #include <string> +#include <chrono> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> @@ -36,12 +37,21 @@ BENCHMARK(CODE_TO_TIME); */ namespace benchmark { + // In case high_resolution_clock is steady, prefer that, otherwise use steady_clock. + struct best_clock { + using hi_res_clock = std::chrono::high_resolution_clock; + using steady_clock = std::chrono::steady_clock; + using type = std::conditional<hi_res_clock::is_steady, hi_res_clock, steady_clock>::type; + }; + using clock = best_clock::type; + using time_point = clock::time_point; + using duration = clock::duration; class State { std::string name; - double maxElapsed; - double beginTime; - double lastTime, minTime, maxTime; + duration maxElapsed; + time_point beginTime, lastTime; + duration minTime, maxTime; uint64_t count; uint64_t countMask; uint64_t beginCycles; @@ -49,12 +59,17 @@ namespace benchmark { uint64_t minCycles; uint64_t maxCycles; public: - State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { - minTime = std::numeric_limits<double>::max(); - maxTime = std::numeric_limits<double>::min(); - minCycles = std::numeric_limits<uint64_t>::max(); - maxCycles = std::numeric_limits<uint64_t>::min(); - countMask = 1; + State(std::string _name, duration _maxElapsed) : + name(_name), + maxElapsed(_maxElapsed), + minTime(duration::max()), + maxTime(duration::zero()), + count(0), + countMask(1), + beginCycles(0), + lastCycles(0), + minCycles(std::numeric_limits<uint64_t>::max()), + maxCycles(std::numeric_limits<uint64_t>::min()) { } bool KeepRunning(); }; @@ -69,7 +84,7 @@ namespace benchmark { public: BenchRunner(std::string name, BenchFunction func); - static void RunAll(double elapsedTimeForOne=1.0); + static void RunAll(duration elapsedTimeForOne = std::chrono::seconds(1)); }; } diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 37fd772435..84e51d809a 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "crypto/sha256.h" -#include "key.h" -#include "validation.h" -#include "util.h" -#include "random.h" +#include <crypto/sha256.h> +#include <key.h> +#include <validation.h> +#include <util.h> +#include <random.h> int main(int argc, char** argv) diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 5aab3381fd..89ba3d3d21 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "coins.h" -#include "policy/policy.h" -#include "wallet/crypter.h" +#include <bench/bench.h> +#include <coins.h> +#include <policy/policy.h> +#include <wallet/crypter.h> #include <vector> diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 7bb1b93668..9533b3c711 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -2,15 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "chainparams.h" -#include "validation.h" -#include "streams.h" -#include "consensus/validation.h" +#include <chainparams.h> +#include <validation.h> +#include <streams.h> +#include <consensus/validation.h> namespace block_bench { -#include "bench/data/block413567.raw.h" +#include <bench/data/block413567.raw.h> } // namespace block_bench // These are the two major time-sinks which happen after we have fully received diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index b7ae5c2d57..35750aa1b6 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "util.h" -#include "validation.h" -#include "checkqueue.h" -#include "prevector.h" +#include <bench/bench.h> +#include <util.h> +#include <validation.h> +#include <checkqueue.h> +#include <prevector.h> #include <vector> #include <boost/thread/thread.hpp> -#include "random.h" +#include <random.h> // This Benchmark tests the CheckQueue with the lightest @@ -19,7 +19,7 @@ static const int MIN_CORES = 2; static const size_t BATCHES = 101; static const size_t BATCH_SIZE = 30; static const int PREVECTOR_SIZE = 28; -static const int QUEUE_BATCH_SIZE = 128; +static const unsigned int QUEUE_BATCH_SIZE = 128; static void CCheckQueueSpeed(benchmark::State& state) { struct FakeJobNoWork { diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f8956508f6..ff57f88170 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "wallet/wallet.h" +#include <bench/bench.h> +#include <wallet/wallet.h> #include <set> diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 410a08e512..b37b5cad62 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -4,16 +4,16 @@ #include <iostream> -#include "bench.h" -#include "bloom.h" -#include "hash.h" -#include "random.h" -#include "uint256.h" -#include "utiltime.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "crypto/sha512.h" +#include <bench/bench.h> +#include <bloom.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> +#include <utiltime.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index c6a05567be..b0bfa95144 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "support/lockedpool.h" +#include <support/lockedpool.h> #include <iostream> #include <vector> diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 073bbde016..eda6edbb23 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "policy/policy.h" -#include "txmempool.h" +#include <bench/bench.h> +#include <policy/policy.h> +#include <txmempool.h> #include <list> #include <vector> diff --git a/src/bench/perf.cpp b/src/bench/perf.cpp index a549ec29ea..4045670a31 100644 --- a/src/bench/perf.cpp +++ b/src/bench/perf.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "perf.h" +#include <bench/perf.h> #if defined(__i386__) || defined(__x86_64__) diff --git a/src/bench/prevector_destructor.cpp b/src/bench/prevector_destructor.cpp index 55af3de4fe..de7ecab737 100644 --- a/src/bench/prevector_destructor.cpp +++ b/src/bench/prevector_destructor.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "prevector.h" +#include <bench/bench.h> +#include <prevector.h> static void PrevectorDestructor(benchmark::State& state) { diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 73c02cf718..452099b800 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -4,9 +4,8 @@ #include <iostream> -#include "bench.h" -#include "bloom.h" -#include "utiltime.h" +#include <bench/bench.h> +#include <bloom.h> static void RollingBloom(benchmark::State& state) { @@ -23,10 +22,10 @@ static void RollingBloom(benchmark::State& state) data[2] = count >> 16; data[3] = count >> 24; if (countnow == nEntriesPerGeneration) { - int64_t b = GetTimeMicros(); + auto b = benchmark::clock::now(); filter.insert(data); - int64_t e = GetTimeMicros(); - std::cout << "RollingBloom-refresh,1," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "\n"; + auto total = std::chrono::duration_cast<std::chrono::nanoseconds>(benchmark::clock::now() - b).count(); + std::cout << "RollingBloom-refresh,1," << total << "," << total << "," << total << "\n"; countnow = 0; } else { filter.insert(data); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index ef7381c120..bfa5806c9d 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "key.h" +#include <bench/bench.h> +#include <key.h> #if defined(HAVE_CONSENSUS_LIB) -#include "script/bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> #endif -#include "script/script.h" -#include "script/sign.h" -#include "streams.h" +#include <script/script.h> +#include <script/sign.h> +#include <streams.h> #include <array> diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index e21a269221..136981b709 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -4,22 +4,22 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparamsbase.h" -#include "clientversion.h" -#include "fs.h" -#include "rpc/client.h" -#include "rpc/protocol.h" -#include "util.h" -#include "utilstrencodings.h" +#include <chainparamsbase.h> +#include <clientversion.h> +#include <fs.h> +#include <rpc/client.h> +#include <rpc/protocol.h> +#include <util.h> +#include <utilstrencodings.h> #include <stdio.h> #include <event2/buffer.h> #include <event2/keyvalq_struct.h> -#include "support/events.h" +#include <support/events.h> #include <univalue.h> @@ -213,6 +213,9 @@ public: /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override { + if (!args.empty()) { + throw std::runtime_error("-getinfo takes no arguments"); + } UniValue result(UniValue::VARR); result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e4f44435ba..c9223dbf10 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -3,24 +3,24 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "base58.h" -#include "clientversion.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "core_io.h" -#include "keystore.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "script/sign.h" +#include <base58.h> +#include <clientversion.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <core_io.h> +#include <keystore.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/sign.h> #include <univalue.h> -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> #include <stdio.h> @@ -387,6 +387,10 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s scriptPubKey = GetScriptForWitness(scriptPubKey); } if (bScriptHash) { + if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { + throw std::runtime_error(strprintf( + "redeemScript exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_ELEMENT_SIZE)); + } // Get the ID for the script, and then construct a P2SH destination for it. scriptPubKey = GetScriptForDestination(CScriptID(scriptPubKey)); } @@ -447,10 +451,19 @@ static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& str bScriptHash = (flags.find("S") != std::string::npos); } + if (scriptPubKey.size() > MAX_SCRIPT_SIZE) { + throw std::runtime_error(strprintf( + "script exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_SIZE)); + } + if (bSegWit) { scriptPubKey = GetScriptForWitness(scriptPubKey); } if (bScriptHash) { + if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { + throw std::runtime_error(strprintf( + "redeemScript exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_ELEMENT_SIZE)); + } scriptPubKey = GetScriptForDestination(CScriptID(scriptPubKey)); } @@ -690,10 +703,10 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); else if (command == "outpubkey") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxAddOutPubKey(tx, commandVal); } else if (command == "outmultisig") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxAddOutMultiSig(tx, commandVal); } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); @@ -701,7 +714,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxAddOutData(tx, commandVal); else if (command == "sign") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxSign(tx, commandVal); } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 543eba0e69..bc23912d2b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -4,21 +4,21 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparams.h" -#include "clientversion.h" -#include "compat.h" -#include "fs.h" -#include "rpc/server.h" -#include "init.h" -#include "noui.h" -#include "scheduler.h" -#include "util.h" -#include "httpserver.h" -#include "httprpc.h" -#include "utilstrencodings.h" +#include <chainparams.h> +#include <clientversion.h> +#include <compat.h> +#include <fs.h> +#include <rpc/server.h> +#include <init.h> +#include <noui.h> +#include <scheduler.h> +#include <util.h> +#include <httpserver.h> +#include <httprpc.h> +#include <utilstrencodings.h> #include <boost/thread.hpp> @@ -120,7 +120,7 @@ bool AppInit(int argc, char* argv[]) for (int i = 1; i < argc; i++) { if (!IsSwitchChar(argv[i][0])) { fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); - exit(EXIT_FAILURE); + return false; } } @@ -132,17 +132,17 @@ bool AppInit(int argc, char* argv[]) if (!AppInitBasicSetup()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (!AppInitParameterInteraction()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (!AppInitSanityChecks()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (gArgs.GetBoolArg("-daemon", false)) { @@ -163,7 +163,7 @@ bool AppInit(int argc, char* argv[]) if (!AppInitLockDataDirectory()) { // If locking the data directory failed, exit immediately - exit(EXIT_FAILURE); + return false; } fRet = AppInitMain(threadGroup, scheduler); } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 6f27b7b9dc..6ff4d97bf4 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -2,16 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "blockencodings.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "chainparams.h" -#include "hash.h" -#include "random.h" -#include "streams.h" -#include "txmempool.h" -#include "validation.h" -#include "util.h" +#include <blockencodings.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <chainparams.h> +#include <hash.h> +#include <random.h> +#include <streams.h> +#include <txmempool.h> +#include <validation.h> +#include <util.h> #include <unordered_map> diff --git a/src/blockencodings.h b/src/blockencodings.h index 50478f9f32..d034bb3080 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_BLOCK_ENCODINGS_H #define BITCOIN_BLOCK_ENCODINGS_H -#include "primitives/block.h" +#include <primitives/block.h> #include <memory> diff --git a/src/bloom.cpp b/src/bloom.cpp index fa884f0bf3..11ea099637 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bloom.h" - -#include "primitives/transaction.h" -#include "hash.h" -#include "script/script.h" -#include "script/standard.h" -#include "random.h" -#include "streams.h" +#include <bloom.h> + +#include <primitives/transaction.h> +#include <hash.h> +#include <script/script.h> +#include <script/standard.h> +#include <random.h> +#include <streams.h> #include <math.h> #include <stdlib.h> diff --git a/src/bloom.h b/src/bloom.h index 7ca9682239..d29a8dcd6f 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_BLOOM_H #define BITCOIN_BLOOM_H -#include "serialize.h" +#include <serialize.h> #include <vector> diff --git a/src/chain.cpp b/src/chain.cpp index 5e3dd9b31b..7ebc08a50b 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" +#include <chain.h> /** * CChain implementation @@ -80,12 +80,13 @@ int static inline GetSkipHeight(int height) { return (height & 1) ? InvertLowestOne(InvertLowestOne(height - 1)) + 1 : InvertLowestOne(height); } -CBlockIndex* CBlockIndex::GetAncestor(int height) +const CBlockIndex* CBlockIndex::GetAncestor(int height) const { - if (height > nHeight || height < 0) + if (height > nHeight || height < 0) { return nullptr; + } - CBlockIndex* pindexWalk = this; + const CBlockIndex* pindexWalk = this; int heightWalk = nHeight; while (heightWalk > height) { int heightSkip = GetSkipHeight(heightWalk); @@ -106,9 +107,9 @@ CBlockIndex* CBlockIndex::GetAncestor(int height) return pindexWalk; } -const CBlockIndex* CBlockIndex::GetAncestor(int height) const +CBlockIndex* CBlockIndex::GetAncestor(int height) { - return const_cast<CBlockIndex*>(this)->GetAncestor(height); + return const_cast<CBlockIndex*>(static_cast<const CBlockIndex*>(this)->GetAncestor(height)); } void CBlockIndex::BuildSkip() diff --git a/src/chain.h b/src/chain.h index f1036e5d92..ca1900c22b 100644 --- a/src/chain.h +++ b/src/chain.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_CHAIN_H #define BITCOIN_CHAIN_H -#include "arith_uint256.h" -#include "primitives/block.h" -#include "pow.h" -#include "tinyformat.h" -#include "uint256.h" +#include <arith_uint256.h> +#include <primitives/block.h> +#include <pow.h> +#include <tinyformat.h> +#include <uint256.h> #include <vector> @@ -304,7 +304,7 @@ public: return (int64_t)nTimeMax; } - enum { nMedianTimeSpan=11 }; + static constexpr int nMedianTimeSpan = 11; int64_t GetMedianTimePast() const { diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 85c9cd6934..a78f530036 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -3,16 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "consensus/merkle.h" +#include <chainparams.h> +#include <consensus/merkle.h> -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> #include <assert.h> -#include "chainparamsseeds.h" +#include <chainparamsseeds.h> static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { @@ -75,6 +75,7 @@ public: CMainParams() { strNetworkID = "main"; consensus.nSubsidyHalvingInterval = 210000; + consensus.BIP16Height = 173805; // 00000000000000ce80a7e057163a4db1d5ad7b20fb6f598c9597b9665c8fb0d4 - April 1, 2012 consensus.BIP34Height = 227931; consensus.BIP34Hash = uint256S("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"); consensus.BIP65Height = 388381; // 000000000000000004c2b624ed5d7756c508d90fd0da2c7c679febfa6c4735f0 @@ -145,7 +146,7 @@ public: fRequireStandard = true; fMineBlocksOnDemand = false; - checkpointData = (CCheckpointData) { + checkpointData = { { { 11111, uint256S("0x0000000069e244f73d78e8fd29ba2fd2ed618bd6fa2ee92559f542fdb26e7c1d")}, { 33333, uint256S("0x000000002dd5588a74784eaa7ab0507a18ad16a236e7b1ce69f00d7ddfb5d0a6")}, @@ -181,6 +182,7 @@ public: CTestNetParams() { strNetworkID = "test"; consensus.nSubsidyHalvingInterval = 210000; + consensus.BIP16Height = 514; // 00000000040b4e986385315e14bee30ad876d8b47f748025b26683116d21aa65 consensus.BIP34Height = 21111; consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"); consensus.BIP65Height = 581885; // 00000000007f6655f22f98e72ed80d8b06dc761d5da09df0fa1dc4be4f861eb6 @@ -229,8 +231,8 @@ public: // nodes with support for servicebits filtering should be at the top vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch", true); vSeeds.emplace_back("seed.tbtc.petertodd.org", true); + vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl", true); vSeeds.emplace_back("testnet-seed.bluematt.me", false); - vSeeds.emplace_back("testnet-seed.bitcoin.schildbach.de", false); base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); @@ -247,7 +249,7 @@ public: fMineBlocksOnDemand = false; - checkpointData = (CCheckpointData) { + checkpointData = { { {546, uint256S("000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")}, } @@ -271,6 +273,7 @@ public: CRegTestParams() { strNetworkID = "regtest"; consensus.nSubsidyHalvingInterval = 150; + consensus.BIP16Height = 0; // always enforce P2SH BIP16 on regtest consensus.BIP34Height = 100000000; // BIP34 has not activated on regtest (far in the future so block v1 are not rejected in tests) consensus.BIP34Hash = uint256(); consensus.BIP65Height = 1351; // BIP65 activated on regtest (Used in rpc activation tests) @@ -284,13 +287,13 @@ public: consensus.nMinerConfirmationWindow = 144; // Faster than normal for regtest (144 instead of 2016) consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit = 28; consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; consensus.vDeployments[Consensus::DEPLOYMENT_CSV].bit = 0; consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1; - consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = Consensus::BIP9Deployment::ALWAYS_ACTIVE; + consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; // The best chain should have at least this much work. consensus.nMinimumChainWork = uint256S("0x00"); @@ -317,7 +320,7 @@ public: fRequireStandard = false; fMineBlocksOnDemand = true; - checkpointData = (CCheckpointData) { + checkpointData = { { {0, uint256S("0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206")}, } diff --git a/src/chainparams.h b/src/chainparams.h index 3948c9163f..0d82a998b1 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_CHAINPARAMS_H #define BITCOIN_CHAINPARAMS_H -#include "chainparamsbase.h" -#include "consensus/params.h" -#include "primitives/block.h" -#include "protocol.h" +#include <chainparamsbase.h> +#include <consensus/params.h> +#include <primitives/block.h> +#include <protocol.h> #include <memory> #include <vector> diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index c966683b72..a7e87faee3 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -3,10 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparamsbase.h" +#include <chainparamsbase.h> -#include "tinyformat.h" -#include "util.h" +#include <tinyformat.h> +#include <util.h> #include <assert.h> diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index 5afe3e66b6..929d864588 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -2,13 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "checkpoints.h" +#include <checkpoints.h> -#include "chain.h" -#include "chainparams.h" -#include "reverse_iterator.h" -#include "validation.h" -#include "uint256.h" +#include <chain.h> +#include <chainparams.h> +#include <reverse_iterator.h> +#include <validation.h> #include <stdint.h> diff --git a/src/checkpoints.h b/src/checkpoints.h index 7449d1063b..df97a674a8 100644 --- a/src/checkpoints.h +++ b/src/checkpoints.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CHECKPOINTS_H #define BITCOIN_CHECKPOINTS_H -#include "uint256.h" +#include <uint256.h> #include <map> diff --git a/src/checkqueue.h b/src/checkqueue.h index 6377fbe942..08de336079 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H -#include "sync.h" +#include <sync.h> #include <algorithm> #include <vector> diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 8a4b408831..1934e71bbc 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -2,11 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "clientversion.h" +#include <clientversion.h> -#include "tinyformat.h" +#include <tinyformat.h> -#include <string> /** * Name of client reported in the 'version' message. Report the same name @@ -39,7 +38,7 @@ const std::string CLIENT_NAME("Satoshi"); //! First, include build.h if requested #ifdef HAVE_BUILD_INFO -#include "build.h" +#include <obj/build.h> #endif //! git will put "#define GIT_ARCHIVE 1" on the next line inside archives. $Format:%n#define GIT_ARCHIVE 1$ diff --git a/src/clientversion.h b/src/clientversion.h index 3d5392619b..3a3d117d53 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -6,7 +6,7 @@ #define BITCOIN_CLIENTVERSION_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H // Check that required client information is defined diff --git a/src/coins.cpp b/src/coins.cpp index b45dfc3342..afaf5ccc13 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -2,13 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coins.h" +#include <coins.h> -#include "consensus/consensus.h" -#include "memusage.h" -#include "random.h" - -#include <assert.h> +#include <consensus/consensus.h> +#include <random.h> bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } @@ -146,56 +143,58 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { - if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). - CCoinsMap::iterator itUs = cacheCoins.find(it->first); - if (itUs == cacheCoins.end()) { - // The parent cache does not have an entry, while the child does - // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { - // Otherwise we will need to create it in the parent - // and move the data up and mark it as dirty - CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coin = std::move(it->second.coin); - cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); - entry.flags = CCoinsCacheEntry::DIRTY; - // We can mark it FRESH in the parent if it was FRESH in the child - // Otherwise it might have just been flushed from the parent's cache - // and already exist in the grandparent - if (it->second.flags & CCoinsCacheEntry::FRESH) - entry.flags |= CCoinsCacheEntry::FRESH; + for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) { + // Ignore non-dirty entries (optimization). + if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) { + continue; + } + CCoinsMap::iterator itUs = cacheCoins.find(it->first); + if (itUs == cacheCoins.end()) { + // The parent cache does not have an entry, while the child does + // We can ignore it if it's both FRESH and pruned in the child + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { + // Otherwise we will need to create it in the parent + // and move the data up and mark it as dirty + CCoinsCacheEntry& entry = cacheCoins[it->first]; + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); + entry.flags = CCoinsCacheEntry::DIRTY; + // We can mark it FRESH in the parent if it was FRESH in the child + // Otherwise it might have just been flushed from the parent's cache + // and already exist in the grandparent + if (it->second.flags & CCoinsCacheEntry::FRESH) { + entry.flags |= CCoinsCacheEntry::FRESH; } - } else { - // Assert that the child cache entry was not marked FRESH if the - // parent cache entry has unspent outputs. If this ever happens, - // it means the FRESH flag was misapplied and there is a logic - // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) - throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + } + } else { + // Assert that the child cache entry was not marked FRESH if the + // parent cache entry has unspent outputs. If this ever happens, + // it means the FRESH flag was misapplied and there is a logic + // error in the calling code. + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) { + throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + } - // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { - // The grandparent does not have an entry, and the child is - // modified and being pruned. This means we can just delete - // it from the parent. - cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); - cacheCoins.erase(itUs); - } else { - // A normal modification. - cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); - itUs->second.coin = std::move(it->second.coin); - cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); - itUs->second.flags |= CCoinsCacheEntry::DIRTY; - // NOTE: It is possible the child has a FRESH flag here in - // the event the entry we found in the parent is pruned. But - // we must not copy that FRESH flag to the parent as that - // pruned state likely still needs to be communicated to the - // grandparent. - } + // Found the entry in the parent cache + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { + // The grandparent does not have an entry, and the child is + // modified and being pruned. This means we can just delete + // it from the parent. + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + cacheCoins.erase(itUs); + } else { + // A normal modification. + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); + itUs->second.flags |= CCoinsCacheEntry::DIRTY; + // NOTE: It is possible the child has a FRESH flag here in + // the event the entry we found in the parent is pruned. But + // we must not copy that FRESH flag to the parent as that + // pruned state likely still needs to be communicated to the + // grandparent. } } - CCoinsMap::iterator itOld = it++; - mapCoins.erase(itOld); } hashBlock = hashBlockIn; return true; diff --git a/src/coins.h b/src/coins.h index 181b2fd4b9..0280abb56c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,13 +6,13 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H -#include "primitives/transaction.h" -#include "compressor.h" -#include "core_memusage.h" -#include "hash.h" -#include "memusage.h" -#include "serialize.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <compressor.h> +#include <core_memusage.h> +#include <hash.h> +#include <memusage.h> +#include <serialize.h> +#include <uint256.h> #include <assert.h> #include <stdint.h> diff --git a/src/compat.h b/src/compat.h index e022659c01..89cc4709a4 100644 --- a/src/compat.h +++ b/src/compat.h @@ -7,7 +7,7 @@ #define BITCOIN_COMPAT_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #ifdef WIN32 @@ -31,6 +31,7 @@ #include <mswsock.h> #include <windows.h> #include <ws2tcpip.h> +#include <stdint.h> #else #include <sys/fcntl.h> #include <sys/mman.h> @@ -49,7 +50,7 @@ #ifndef WIN32 typedef unsigned int SOCKET; -#include "errno.h" +#include <errno.h> #define WSAGetLastError() errno #define WSAEINVAL EINVAL #define WSAEALREADY EALREADY @@ -71,6 +72,15 @@ typedef unsigned int SOCKET; #else #define MAX_PATH 1024 #endif +#ifdef _MSC_VER +#if !defined(ssize_t) +#ifdef _WIN64 +typedef int64_t ssize_t; +#else +typedef int32_t ssize_t; +#endif +#endif +#endif #if HAVE_DECL_STRNLEN == 0 size_t strnlen( const char *start, size_t max_len); diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index d93ff7413a..a6df6ded7a 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -6,7 +6,7 @@ #define BITCOIN_COMPAT_BYTESWAP_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdint.h> @@ -37,7 +37,7 @@ inline uint16_t bswap_16(uint16_t x) { return (x >> 8) | (x << 8); } -#endif // HAVE_DECL_BSWAP16 +#endif // HAVE_DECL_BSWAP16 == 0 #if HAVE_DECL_BSWAP_32 == 0 inline uint32_t bswap_32(uint32_t x) @@ -45,7 +45,7 @@ inline uint32_t bswap_32(uint32_t x) return (((x & 0xff000000U) >> 24) | ((x & 0x00ff0000U) >> 8) | ((x & 0x0000ff00U) << 8) | ((x & 0x000000ffU) << 24)); } -#endif // HAVE_DECL_BSWAP32 +#endif // HAVE_DECL_BSWAP32 == 0 #if HAVE_DECL_BSWAP_64 == 0 inline uint64_t bswap_64(uint64_t x) @@ -59,7 +59,7 @@ inline uint64_t bswap_64(uint64_t x) | ((x & 0x000000000000ff00ull) << 40) | ((x & 0x00000000000000ffull) << 56)); } -#endif // HAVE_DECL_BSWAP64 +#endif // HAVE_DECL_BSWAP64 == 0 #endif // defined(__APPLE__) diff --git a/src/compat/endian.h b/src/compat/endian.h index dbf178f53c..9fabbd0bfb 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -6,10 +6,10 @@ #define BITCOIN_COMPAT_ENDIAN_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat/byteswap.h" +#include <compat/byteswap.h> #include <stdint.h> diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 3b9c70df7f..6633658c16 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstddef> diff --git a/src/compat/glibc_sanity.cpp b/src/compat/glibc_sanity.cpp index b4d1c90992..20d2ad3cb6 100644 --- a/src/compat/glibc_sanity.cpp +++ b/src/compat/glibc_sanity.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstddef> diff --git a/src/compat/strnlen.cpp b/src/compat/strnlen.cpp index 1ac266c2d1..313435a222 100644 --- a/src/compat/strnlen.cpp +++ b/src/compat/strnlen.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstring> diff --git a/src/compressor.cpp b/src/compressor.cpp index f4c12f38d2..a729e28f07 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compressor.h" +#include <compressor.h> -#include "hash.h" -#include "pubkey.h" -#include "script/standard.h" +#include <hash.h> +#include <pubkey.h> +#include <script/standard.h> bool CScriptCompressor::IsToKeyID(CKeyID &hash) const { diff --git a/src/compressor.h b/src/compressor.h index 094c1bcfe1..8731eef9f7 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_COMPRESSOR_H #define BITCOIN_COMPRESSOR_H -#include "primitives/transaction.h" -#include "script/script.h" -#include "serialize.h" +#include <primitives/transaction.h> +#include <script/script.h> +#include <serialize.h> class CKeyID; class CPubKey; diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index ddd4ee9fab..6e3bac2d0e 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -24,12 +24,9 @@ static const size_t MIN_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 60; // 60 is static const size_t MIN_SERIALIZABLE_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 10; // 10 is the lower bound for the size of a serialized CTransaction /** Flags for nSequence and nLockTime locks */ -enum { - /* Interpret sequence numbers as relative lock-time constraints. */ - LOCKTIME_VERIFY_SEQUENCE = (1 << 0), - - /* Use GetMedianTimePast() instead of nTime for end point timestamp. */ - LOCKTIME_MEDIAN_TIME_PAST = (1 << 1), -}; +/** Interpret sequence numbers as relative lock-time constraints. */ +static constexpr unsigned int LOCKTIME_VERIFY_SEQUENCE = (1 << 0); +/** Use GetMedianTimePast() instead of nTime for end point timestamp. */ +static constexpr unsigned int LOCKTIME_MEDIAN_TIME_PAST = (1 << 1); #endif // BITCOIN_CONSENSUS_CONSENSUS_H diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 798ce4b5fd..fef4a5d560 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkle.h" -#include "hash.h" -#include "utilstrencodings.h" +#include <consensus/merkle.h> +#include <hash.h> +#include <utilstrencodings.h> /* WARNING! If you're reading this because you're learning about crypto and/or designing a new system that will use merkle trees, keep in mind diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 33764c7460..c1573cc214 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -8,9 +8,9 @@ #include <stdint.h> #include <vector> -#include "primitives/transaction.h" -#include "primitives/block.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <primitives/block.h> +#include <uint256.h> uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = nullptr); std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position); diff --git a/src/consensus/params.h b/src/consensus/params.h index 6240e82857..c9fbba12a2 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -6,7 +6,8 @@ #ifndef BITCOIN_CONSENSUS_PARAMS_H #define BITCOIN_CONSENSUS_PARAMS_H -#include "uint256.h" +#include <uint256.h> +#include <limits> #include <map> #include <string> @@ -31,6 +32,15 @@ struct BIP9Deployment { int64_t nStartTime; /** Timeout/expiry MedianTime for the deployment attempt. */ int64_t nTimeout; + + /** Constant for nTimeout very far in the future. */ + static constexpr int64_t NO_TIMEOUT = std::numeric_limits<int64_t>::max(); + + /** Special value for nStartTime indicating that the deployment is always active. + * This is useful for testing, as it means tests don't need to deal with the activation + * process (which takes at least 3 BIP9 intervals). Only tests that specifically test the + * behaviour during activation cannot use this. */ + static constexpr int64_t ALWAYS_ACTIVE = -1; }; /** @@ -39,6 +49,8 @@ struct BIP9Deployment { struct Params { uint256 hashGenesisBlock; int nSubsidyHalvingInterval; + /** Block height at which BIP16 becomes active */ + int BIP16Height; /** Block height and hash at which BIP34 becomes active */ int BIP34Height; uint256 BIP34Hash; diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 70aa9d7006..be73d0a2f9 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -2,17 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "tx_verify.h" +#include <consensus/tx_verify.h> -#include "consensus.h" -#include "primitives/transaction.h" -#include "script/interpreter.h" -#include "validation.h" +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <script/interpreter.h> +#include <consensus/validation.h> // TODO remove the following dependencies -#include "chain.h" -#include "coins.h" -#include "utilmoneystr.h" +#include <chain.h> +#include <coins.h> +#include <utilmoneystr.h> bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 288892462d..a4836ceda4 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CONSENSUS_TX_VERIFY_H #define BITCOIN_CONSENSUS_TX_VERIFY_H -#include "amount.h" +#include <amount.h> #include <stdint.h> #include <vector> diff --git a/src/consensus/validation.h b/src/consensus/validation.h index b6740c9d9f..55f6c363fc 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -7,10 +7,10 @@ #define BITCOIN_CONSENSUS_VALIDATION_H #include <string> -#include "version.h" -#include "consensus/consensus.h" -#include "primitives/transaction.h" -#include "primitives/block.h" +#include <version.h> +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <primitives/block.h> /** "reject" message codes */ static const unsigned char REJECT_MALFORMED = 0x01; diff --git a/src/core_io.h b/src/core_io.h index ccc72ebb32..3cdae06dfb 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CORE_IO_H #define BITCOIN_CORE_IO_H -#include "amount.h" +#include <amount.h> #include <string> #include <vector> @@ -20,7 +20,7 @@ class UniValue; // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); -bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); +bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); diff --git a/src/core_memusage.h b/src/core_memusage.h index f038e7b154..d234f0a9d0 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_CORE_MEMUSAGE_H #define BITCOIN_CORE_MEMUSAGE_H -#include "primitives/transaction.h" -#include "primitives/block.h" -#include "memusage.h" +#include <primitives/transaction.h> +#include <primitives/block.h> +#include <memusage.h> static inline size_t RecursiveDynamicUsage(const CScript& script) { return memusage::DynamicUsage(script); diff --git a/src/core_read.cpp b/src/core_read.cpp index 7018131a13..4f2dabe10f 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -2,17 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "core_io.h" +#include <core_io.h> -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "serialize.h" -#include "streams.h" +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <serialize.h> +#include <streams.h> #include <univalue.h> -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/predicate.hpp> @@ -108,39 +108,39 @@ bool CheckTxScriptsSanity(const CMutableTransaction& tx) return true; } -bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) +bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness, bool try_witness) { - if (!IsHex(strHexTx)) { + if (!IsHex(hex_tx)) { return false; } - std::vector<unsigned char> txData(ParseHex(strHexTx)); + std::vector<unsigned char> txData(ParseHex(hex_tx)); - if (fTryNoWitness) { + if (try_no_witness) { CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); try { ssData >> tx; - if (ssData.eof() && CheckTxScriptsSanity(tx)) { + if (ssData.eof() && (!try_witness || CheckTxScriptsSanity(tx))) { return true; } - } - catch (const std::exception&) { + } catch (const std::exception&) { // Fall through. } } - CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION); - try { - ssData >> tx; - if (!ssData.empty()) { - return false; + if (try_witness) { + CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION); + try { + ssData >> tx; + if (ssData.empty()) { + return true; + } + } catch (const std::exception&) { + // Fall through. } } - catch (const std::exception&) { - return false; - } - - return true; + + return false; } bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) diff --git a/src/core_write.cpp b/src/core_write.cpp index e16db13650..e97876bef3 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -2,19 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "core_io.h" - -#include "base58.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "script/script.h" -#include "script/standard.h" -#include "serialize.h" -#include "streams.h" +#include <core_io.h> + +#include <base58.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <script/script.h> +#include <script/standard.h> +#include <serialize.h> +#include <streams.h> #include <univalue.h> -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> UniValue ValueFromAmount(const CAmount& amount) { diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 5e70d25eee..baba8bcad0 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "aes.h" -#include "crypto/common.h" +#include <crypto/aes.h> +#include <crypto/common.h> #include <assert.h> #include <string.h> extern "C" { -#include "crypto/ctaes/ctaes.c" +#include <crypto/ctaes/ctaes.c> } AES128Encrypt::AES128Encrypt(const unsigned char key[16]) diff --git a/src/crypto/aes.h b/src/crypto/aes.h index a7b63b19df..7dda94d291 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -8,7 +8,7 @@ #define BITCOIN_CRYPTO_AES_H extern "C" { -#include "crypto/ctaes/ctaes.h" +#include <crypto/ctaes/ctaes.h> } static const int AES_BLOCKSIZE = 16; diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 4038ae9f86..ac4470f04f 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -5,8 +5,8 @@ // Based on the public domain implementation 'merged' by D. J. Bernstein // See https://cr.yp.to/chacha.html. -#include "crypto/common.h" -#include "crypto/chacha20.h" +#include <crypto/common.h> +#include <crypto/chacha20.h> #include <string.h> diff --git a/src/crypto/common.h b/src/crypto/common.h index bd9bc9420b..5a193a4e64 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -6,13 +6,13 @@ #define BITCOIN_CRYPTO_COMMON_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdint.h> #include <string.h> -#include "compat/endian.h" +#include <compat/endian.h> uint16_t static inline ReadLE16(const unsigned char* ptr) { diff --git a/src/crypto/hmac_sha256.cpp b/src/crypto/hmac_sha256.cpp index 3c791625d0..9366e216c2 100644 --- a/src/crypto/hmac_sha256.cpp +++ b/src/crypto/hmac_sha256.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/hmac_sha256.h" +#include <crypto/hmac_sha256.h> #include <string.h> diff --git a/src/crypto/hmac_sha256.h b/src/crypto/hmac_sha256.h index 8c42fcfe14..40d40f9fb2 100644 --- a/src/crypto/hmac_sha256.h +++ b/src/crypto/hmac_sha256.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CRYPTO_HMAC_SHA256_H #define BITCOIN_CRYPTO_HMAC_SHA256_H -#include "crypto/sha256.h" +#include <crypto/sha256.h> #include <stdint.h> #include <stdlib.h> diff --git a/src/crypto/hmac_sha512.cpp b/src/crypto/hmac_sha512.cpp index 5939c6ec47..f984201f6a 100644 --- a/src/crypto/hmac_sha512.cpp +++ b/src/crypto/hmac_sha512.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/hmac_sha512.h" +#include <crypto/hmac_sha512.h> #include <string.h> diff --git a/src/crypto/hmac_sha512.h b/src/crypto/hmac_sha512.h index 17dee61ea8..6d3fd59f27 100644 --- a/src/crypto/hmac_sha512.h +++ b/src/crypto/hmac_sha512.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CRYPTO_HMAC_SHA512_H #define BITCOIN_CRYPTO_HMAC_SHA512_H -#include "crypto/sha512.h" +#include <crypto/sha512.h> #include <stdint.h> #include <stdlib.h> diff --git a/src/crypto/ripemd160.cpp b/src/crypto/ripemd160.cpp index 77c9acfc26..4afa9abd1e 100644 --- a/src/crypto/ripemd160.cpp +++ b/src/crypto/ripemd160.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/ripemd160.h" +#include <crypto/ripemd160.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/crypto/sha1.cpp b/src/crypto/sha1.cpp index 0b895b33a2..8ccc5e112a 100644 --- a/src/crypto/sha1.cpp +++ b/src/crypto/sha1.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha1.h" +#include <crypto/sha1.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 29afe86ec7..9a21aec20e 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha256.h" -#include "crypto/common.h" +#include <crypto/sha256.h> +#include <crypto/common.h> #include <assert.h> #include <string.h> diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 564127cc31..aa46e7ba86 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha512.h" +#include <crypto/sha512.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index dfc90f3ab9..6ff740f581 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -2,11 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "dbwrapper.h" +#include <dbwrapper.h> -#include "fs.h" -#include "util.h" -#include "random.h" +#include <random.h> #include <leveldb/cache.h> #include <leveldb/env.h> @@ -64,7 +62,7 @@ public: assert(p <= limit); base[std::min(bufsize - 1, (int)(p - base))] = '\0'; - LogPrintStr(base); + LogPrintf("leveldb: %s", base); if (base != buffer) { delete[] base; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index e19fde51c1..9f4d4814eb 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -5,13 +5,13 @@ #ifndef BITCOIN_DBWRAPPER_H #define BITCOIN_DBWRAPPER_H -#include "clientversion.h" -#include "fs.h" -#include "serialize.h" -#include "streams.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <clientversion.h> +#include <fs.h> +#include <serialize.h> +#include <streams.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> diff --git a/src/fs.cpp b/src/fs.cpp index a5e12f1cfc..570ed3e2ee 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -1,4 +1,4 @@ -#include "fs.h" +#include <fs.h> namespace fsbridge { diff --git a/src/hash.cpp b/src/hash.cpp index 5a15600be5..ea61764ded 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -2,10 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "hash.h" -#include "crypto/common.h" -#include "crypto/hmac_sha512.h" -#include "pubkey.h" +#include <hash.h> +#include <crypto/common.h> +#include <crypto/hmac_sha512.h> inline uint32_t ROTL32(uint32_t x, int8_t r) diff --git a/src/hash.h b/src/hash.h index 474b13d65b..55724fb636 100644 --- a/src/hash.h +++ b/src/hash.h @@ -6,12 +6,12 @@ #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H -#include "crypto/ripemd160.h" -#include "crypto/sha256.h" -#include "prevector.h" -#include "serialize.h" -#include "uint256.h" -#include "version.h" +#include <crypto/ripemd160.h> +#include <crypto/sha256.h> +#include <prevector.h> +#include <serialize.h> +#include <uint256.h> +#include <version.h> #include <vector> diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 91f96ef207..6b6849e59b 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -2,19 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "httprpc.h" - -#include "base58.h" -#include "chainparams.h" -#include "httpserver.h" -#include "rpc/protocol.h" -#include "rpc/server.h" -#include "random.h" -#include "sync.h" -#include "util.h" -#include "utilstrencodings.h" -#include "ui_interface.h" -#include "crypto/hmac_sha256.h" +#include <httprpc.h> + +#include <base58.h> +#include <chainparams.h> +#include <httpserver.h> +#include <rpc/protocol.h> +#include <rpc/server.h> +#include <random.h> +#include <sync.h> +#include <util.h> +#include <utilstrencodings.h> +#include <ui_interface.h> +#include <crypto/hmac_sha256.h> #include <stdio.h> #include <boost/algorithm/string.hpp> // boost::trim @@ -62,7 +62,7 @@ private: /* Pre-base64-encoded authentication token */ static std::string strRPCUserColonPass; /* Stored RPC timer interface (for unregistration) */ -static HTTPRPCTimerInterface* httpRPCTimerInterface = nullptr; +static std::unique_ptr<HTTPRPCTimerInterface> httpRPCTimerInterface; static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) { @@ -192,7 +192,7 @@ static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) // array of requests } else if (valRequest.isArray()) - strReply = JSONRPCExecBatch(valRequest.get_array()); + strReply = JSONRPCExecBatch(jreq, valRequest.get_array()); else throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); @@ -238,8 +238,8 @@ bool StartHTTPRPC() RegisterHTTPHandler("/wallet/", false, HTTPReq_JSONRPC); #endif assert(EventBase()); - httpRPCTimerInterface = new HTTPRPCTimerInterface(EventBase()); - RPCSetTimerInterface(httpRPCTimerInterface); + httpRPCTimerInterface = MakeUnique<HTTPRPCTimerInterface>(EventBase()); + RPCSetTimerInterface(httpRPCTimerInterface.get()); return true; } @@ -253,8 +253,7 @@ void StopHTTPRPC() LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (httpRPCTimerInterface) { - RPCUnsetTimerInterface(httpRPCTimerInterface); - delete httpRPCTimerInterface; - httpRPCTimerInterface = nullptr; + RPCUnsetTimerInterface(httpRPCTimerInterface.get()); + httpRPCTimerInterface.reset(); } } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5923871691..7d590f8e1f 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -2,16 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "httpserver.h" +#include <httpserver.h> -#include "chainparamsbase.h" -#include "compat.h" -#include "util.h" -#include "utilstrencodings.h" -#include "netbase.h" -#include "rpc/protocol.h" // For HTTP status codes -#include "sync.h" -#include "ui_interface.h" +#include <chainparamsbase.h> +#include <compat.h> +#include <util.h> +#include <utilstrencodings.h> +#include <netbase.h> +#include <rpc/protocol.h> // For HTTP status codes +#include <sync.h> +#include <ui_interface.h> #include <stdio.h> #include <stdlib.h> @@ -24,10 +24,11 @@ #include <event2/thread.h> #include <event2/buffer.h> +#include <event2/bufferevent.h> #include <event2/util.h> #include <event2/keyvalq_struct.h> -#include "support/events.h" +#include <support/events.h> #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> @@ -239,6 +240,16 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { + // Disable reading to work around a libevent bug, fixed in 2.2.0. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_disable(bev, EV_READ); + } + } + } std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req)); LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", @@ -481,6 +492,8 @@ void StopHTTPServer() } if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); + // Exit the event loop as soon as there are no active events. + event_base_loopexit(eventBase, nullptr); // Give event loop a few seconds to exit (to send back last RPC responses), then break it // Before this was solved with event_base_loopexit, but that didn't work as expected in // at least libevent 2.0.21 and always introduced a delay. In libevent @@ -599,8 +612,21 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); - HTTPEvent* ev = new HTTPEvent(eventBase, true, - std::bind(evhttp_send_reply, req, nStatus, (const char*)nullptr, (struct evbuffer *)nullptr)); + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); ev->trigger(nullptr); replySent = true; req = nullptr; // transferred back to main thread diff --git a/src/init.cpp b/src/init.cpp index 539adc23d5..e4cad01b70 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -4,49 +4,49 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "init.h" - -#include "addrman.h" -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "compat/sanity.h" -#include "consensus/validation.h" -#include "fs.h" -#include "httpserver.h" -#include "httprpc.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "netbase.h" -#include "net.h" -#include "net_processing.h" -#include "policy/feerate.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "rpc/server.h" -#include "rpc/register.h" -#include "rpc/safemode.h" -#include "rpc/blockchain.h" -#include "script/standard.h" -#include "script/sigcache.h" -#include "scheduler.h" -#include "timedata.h" -#include "txdb.h" -#include "txmempool.h" -#include "torcontrol.h" -#include "ui_interface.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validationinterface.h" +#include <init.h> + +#include <addrman.h> +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <compat/sanity.h> +#include <consensus/validation.h> +#include <fs.h> +#include <httpserver.h> +#include <httprpc.h> +#include <key.h> +#include <validation.h> +#include <miner.h> +#include <netbase.h> +#include <net.h> +#include <net_processing.h> +#include <policy/feerate.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <rpc/server.h> +#include <rpc/register.h> +#include <rpc/safemode.h> +#include <rpc/blockchain.h> +#include <script/standard.h> +#include <script/sigcache.h> +#include <scheduler.h> +#include <timedata.h> +#include <txdb.h> +#include <txmempool.h> +#include <torcontrol.h> +#include <ui_interface.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validationinterface.h> #ifdef ENABLE_WALLET -#include "wallet/init.h" +#include <wallet/init.h> #endif -#include "warnings.h" +#include <warnings.h> #include <stdint.h> #include <stdio.h> #include <memory> @@ -64,7 +64,7 @@ #include <openssl/crypto.h> #if ENABLE_ZMQ -#include "zmq/zmqnotificationinterface.h" +#include <zmq/zmqnotificationinterface.h> #endif bool fFeeEstimatesInitialized = false; @@ -152,7 +152,7 @@ public: // Writes do not need similar protection, as failure to write is handled by the caller. }; -static CCoinsViewErrorCatcher *pcoinscatcher = nullptr; +static std::unique_ptr<CCoinsViewErrorCatcher> pcoinscatcher; static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; void Interrupt(boost::thread_group& threadGroup) @@ -193,8 +193,8 @@ void Shutdown() // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. - UnregisterValidationInterface(peerLogic.get()); - if(g_connman) g_connman->Stop(); + if (peerLogic) UnregisterValidationInterface(peerLogic.get()); + if (g_connman) g_connman->Stop(); peerLogic.reset(); g_connman.reset(); @@ -235,14 +235,10 @@ void Shutdown() if (pcoinsTip != nullptr) { FlushStateToDisk(); } - delete pcoinsTip; - pcoinsTip = nullptr; - delete pcoinscatcher; - pcoinscatcher = nullptr; - delete pcoinsdbview; - pcoinsdbview = nullptr; - delete pblocktree; - pblocktree = nullptr; + pcoinsTip.reset(); + pcoinscatcher.reset(); + pcoinsdbview.reset(); + pblocktree.reset(); } #ifdef ENABLE_WALLET StopWallets(); @@ -265,6 +261,7 @@ void Shutdown() #endif UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + GetMainSignals().UnregisterWithMempoolSignals(mempool); #ifdef ENABLE_WALLET CloseWallets(); #endif @@ -345,6 +342,7 @@ std::string HelpMessage(HelpMessageMode mode) if (showDebug) strUsage += HelpMessageOpt("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER)); strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file on startup")); + strUsage += HelpMessageOpt("-debuglogfile=<file>", strprintf(_("Specify location of debug log file: this can be an absolute path or a path relative to the data directory (default: %s)"), DEFAULT_DEBUGLOGFILE)); strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); @@ -369,11 +367,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); - strUsage += HelpMessageOpt("-addnode=<ip>", _("Add a node to connect to and attempt to keep the connection open")); + strUsage += HelpMessageOpt("-addnode=<ip>", _("Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info)")); strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -connect=0 disables automatic connections")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -connect=0 disables automatic connections (the rules for this peer are the same as for -addnode)")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)")); @@ -544,14 +542,14 @@ static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex } static bool fHaveGenesis = false; -static boost::mutex cs_GenesisWait; +static CWaitableCriticalSection cs_GenesisWait; static CConditionVariable condvar_GenesisWait; static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) { if (pBlockIndex != nullptr) { { - boost::unique_lock<boost::mutex> lock_GenesisWait(cs_GenesisWait); + WaitableLock lock_GenesisWait(cs_GenesisWait); fHaveGenesis = true; } condvar_GenesisWait.notify_all(); @@ -588,7 +586,7 @@ void CleanupBlockRevFiles() LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); fs::path blocksdir = GetDataDir() / "blocks"; for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (is_regular_file(*it) && + if (fs::is_regular_file(*it) && it->path().filename().string().length() == 12 && it->path().filename().string().substr(8,4) == ".dat") { @@ -815,11 +813,10 @@ void InitLogging() namespace { // Variables internal to initialization process only -ServiceFlags nRelevantServices = NODE_NETWORK; int nMaxConnections; int nUserMaxConnections; int nFD; -ServiceFlags nLocalServices = NODE_NETWORK; +ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); } // namespace @@ -923,7 +920,8 @@ bool AppInitParameterInteraction() // Special-case: if -debug=0/-nodebug is set, turn off debugging messages const std::vector<std::string> categories = gArgs.GetArgs("-debug"); - if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { + if (std::none_of(categories.begin(), categories.end(), + [](std::string cat){return cat == "0" || cat == "none";})) { for (const auto& cat : categories) { uint32_t flag = 0; if (!GetLogCategory(&flag, &cat)) { @@ -1034,11 +1032,6 @@ bool AppInitParameterInteraction() fPruneMode = true; } - RegisterAllCoreRPCCommands(tableRPC); -#ifdef ENABLE_WALLET - RegisterWalletRPC(tableRPC); -#endif - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -1217,8 +1210,11 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) ShrinkDebugFile(); } - if (fPrintToDebugLog) - OpenDebugLog(); + if (fPrintToDebugLog) { + if (!OpenDebugLog()) { + return InitError(strprintf("Could not open debug log file %s", GetDebugLogPath().string())); + } + } if (!fLogTimestamps) LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); @@ -1241,6 +1237,15 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + GetMainSignals().RegisterWithMempoolSignals(mempool); + + /* Register RPC commands regardless of -server setting so they will be + * available in the GUI RPC console even if external calls are disabled. + */ + RegisterAllCoreRPCCommands(tableRPC); +#ifdef ENABLE_WALLET + RegisterWalletRPC(tableRPC); +#endif /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections @@ -1271,7 +1276,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); CConnman& connman = *g_connman; - peerLogic.reset(new PeerLogicValidation(&connman)); + peerLogic.reset(new PeerLogicValidation(&connman, scheduler)); RegisterValidationInterface(peerLogic.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1407,12 +1412,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) do { try { UnloadBlockIndex(); - delete pcoinsTip; - delete pcoinsdbview; - delete pcoinscatcher; - delete pblocktree; - - pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReset); + pcoinsTip.reset(); + pcoinsdbview.reset(); + pcoinscatcher.reset(); + pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); if (fReset) { pblocktree->WriteReindexing(true); @@ -1463,8 +1466,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // At this point we're either in reindex or we've loaded a useful // block tree into mapBlockIndex! - pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState); - pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); + pcoinsdbview.reset(new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState)); + pcoinscatcher.reset(new CCoinsViewErrorCatcher(pcoinsdbview.get())); // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate @@ -1474,13 +1477,13 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!ReplayBlocks(chainparams, pcoinsdbview)) { + if (!ReplayBlocks(chainparams, pcoinsdbview.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); break; } // The on-disk coinsdb is now in a good state, create the cache - pcoinsTip = new CCoinsViewCache(pcoinscatcher); + pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); if (!is_coinsview_empty) { @@ -1522,7 +1525,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (!CVerifyDB().VerifyDB(chainparams, pcoinsdbview, gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + if (!CVerifyDB().VerifyDB(chainparams, pcoinsdbview.get(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); break; @@ -1604,9 +1607,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // Note that setting NODE_WITNESS is never required: the only downside from not // doing so is that after activation, no upgraded nodes will fetch from you. nLocalServices = ServiceFlags(nLocalServices | NODE_WITNESS); - // Only care about others providing witness capabilities if there is a softfork - // defined. - nRelevantServices = ServiceFlags(nRelevantServices | NODE_WITNESS); } // ********************************************************* Step 10: import blocks @@ -1634,7 +1634,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // Wait for genesis block to be processed { - boost::unique_lock<boost::mutex> lock(cs_GenesisWait); + WaitableLock lock(cs_GenesisWait); while (!fHaveGenesis) { condvar_GenesisWait.wait(lock); } @@ -1663,7 +1663,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nRelevantServices = nRelevantServices; connOptions.nMaxConnections = nMaxConnections; connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; @@ -1727,5 +1726,5 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) StartWallets(scheduler); #endif - return !fRequestShutdown; + return true; } diff --git a/src/key.cpp b/src/key.cpp index 315a3978c8..9893b88220 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" +#include <key.h> -#include "arith_uint256.h" -#include "crypto/common.h" -#include "crypto/hmac_sha512.h" -#include "pubkey.h" -#include "random.h" +#include <arith_uint256.h> +#include <crypto/common.h> +#include <crypto/hmac_sha512.h> +#include <random.h> #include <secp256k1.h> #include <secp256k1_recovery.h> @@ -16,43 +16,62 @@ static secp256k1_context* secp256k1_context_sign = nullptr; /** These functions are taken from the libsecp256k1 distribution and are very ugly. */ + +/** + * This parses a format loosely based on a DER encoding of the ECPrivateKey type from + * section C.4 of SEC 1 <http://www.secg.org/sec1-v2.pdf>, with the following caveats: + * + * * The octet-length of the SEQUENCE must be encoded as 1 or 2 octets. It is not + * required to be encoded as one octet if it is less than 256, as DER would require. + * * The octet-length of the SEQUENCE must not be greater than the remaining + * length of the key encoding, but need not match it (i.e. the encoding may contain + * junk after the encoded SEQUENCE). + * * The privateKey OCTET STRING is zero-filled on the left to 32 octets. + * * Anything after the encoding of the privateKey OCTET STRING is ignored, whether + * or not it is validly encoded DER. + * + * out32 must point to an output buffer of length at least 32 bytes. + */ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *privkey, size_t privkeylen) { const unsigned char *end = privkey + privkeylen; - int lenb = 0; - int len = 0; memset(out32, 0, 32); /* sequence header */ - if (end < privkey+1 || *privkey != 0x30) { + if (end - privkey < 1 || *privkey != 0x30u) { return 0; } privkey++; /* sequence length constructor */ - if (end < privkey+1 || !(*privkey & 0x80)) { + if (end - privkey < 1 || !(*privkey & 0x80u)) { return 0; } - lenb = *privkey & ~0x80; privkey++; + size_t lenb = *privkey & ~0x80u; privkey++; if (lenb < 1 || lenb > 2) { return 0; } - if (end < privkey+lenb) { + if (end - privkey < lenb) { return 0; } /* sequence length */ - len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0); + size_t len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0u); privkey += lenb; - if (end < privkey+len) { + if (end - privkey < len) { return 0; } /* sequence element 0: version number (=1) */ - if (end < privkey+3 || privkey[0] != 0x02 || privkey[1] != 0x01 || privkey[2] != 0x01) { + if (end - privkey < 3 || privkey[0] != 0x02u || privkey[1] != 0x01u || privkey[2] != 0x01u) { return 0; } privkey += 3; /* sequence element 1: octet string, up to 32 bytes */ - if (end < privkey+2 || privkey[0] != 0x04 || privkey[1] > 0x20 || end < privkey+2+privkey[1]) { + if (end - privkey < 2 || privkey[0] != 0x04u) { + return 0; + } + size_t oslen = privkey[1]; + privkey += 2; + if (oslen > 32 || end - privkey < oslen) { return 0; } - memcpy(out32 + 32 - privkey[1], privkey + 2, privkey[1]); + memcpy(out32 + (32 - oslen), privkey, oslen); if (!secp256k1_ec_seckey_verify(ctx, out32)) { memset(out32, 0, 32); return 0; @@ -60,7 +79,18 @@ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *ou return 1; } +/** + * This serializes to a DER encoding of the ECPrivateKey type from section C.4 of SEC 1 + * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are + * included. + * + * privkey must point to an output buffer of length at least CKey::PRIVATE_KEY_SIZE bytes. + * privkeylen must initially be set to the size of the privkey buffer. Upon return it + * will be set to the number of bytes used in the buffer. + * key32 must point to a 32-byte raw private key. + */ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *privkey, size_t *privkeylen, const unsigned char *key32, int compressed) { + assert(*privkeylen >= CKey::PRIVATE_KEY_SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; if (!secp256k1_ec_pubkey_create(ctx, &pubkey, key32)) { @@ -86,10 +116,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = 33; + pubkeylen = CPubKey::COMPRESSED_PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; + assert(*privkeylen == CKey::COMPRESSED_PRIVATE_KEY_SIZE); } else { static const unsigned char begin[] = { 0x30,0x82,0x01,0x13,0x02,0x01,0x01,0x04,0x20 @@ -111,10 +142,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = 65; + pubkeylen = CPubKey::PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_UNCOMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; + assert(*privkeylen == CKey::PRIVATE_KEY_SIZE); } return 1; } @@ -136,8 +168,8 @@ CPrivKey CKey::GetPrivKey() const { CPrivKey privkey; int ret; size_t privkeylen; - privkey.resize(279); - privkeylen = 279; + privkey.resize(PRIVATE_KEY_SIZE); + privkeylen = PRIVATE_KEY_SIZE; ret = ec_privkey_export_der(secp256k1_context_sign, (unsigned char*) privkey.data(), &privkeylen, begin(), fCompressed ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); assert(ret); privkey.resize(privkeylen); @@ -147,7 +179,7 @@ CPrivKey CKey::GetPrivKey() const { CPubKey CKey::GetPubKey() const { assert(fValid); secp256k1_pubkey pubkey; - size_t clen = 65; + size_t clen = CPubKey::PUBLIC_KEY_SIZE; CPubKey result; int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, begin()); assert(ret); @@ -160,8 +192,8 @@ CPubKey CKey::GetPubKey() const { bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, uint32_t test_case) const { if (!fValid) return false; - vchSig.resize(72); - size_t nSigLen = 72; + vchSig.resize(CPubKey::SIGNATURE_SIZE); + size_t nSigLen = CPubKey::SIGNATURE_SIZE; unsigned char extra_entropy[32] = {0}; WriteLE32(extra_entropy, test_case); secp256k1_ecdsa_signature sig; @@ -189,7 +221,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) const { if (!fValid) return false; - vchSig.resize(65); + vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; secp256k1_ecdsa_recoverable_signature sig; int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); @@ -219,10 +251,10 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); - assert(pubkey.begin() + 33 == pubkey.end()); + assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { - assert(begin() + 32 == end()); + assert(size() == 32); BIP32Hash(cc, nChild, 0, begin(), vout.data()); } memcpy(ccChild.begin(), vout.data()+32, 32); @@ -1,39 +1,45 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_KEY_H #define BITCOIN_KEY_H -#include "pubkey.h" -#include "serialize.h" -#include "support/allocators/secure.h" -#include "uint256.h" +#include <pubkey.h> +#include <serialize.h> +#include <support/allocators/secure.h> +#include <uint256.h> #include <stdexcept> #include <vector> /** - * secp256k1: - * const unsigned int PRIVATE_KEY_SIZE = 279; - * const unsigned int PUBLIC_KEY_SIZE = 65; - * const unsigned int SIGNATURE_SIZE = 72; - * - * see www.keylength.com - * script supports up to 75 for single byte push - */ - -/** * secure_allocator is defined in allocators.h - * CPrivKey is a serialized private key, with all parameters included (279 bytes) + * CPrivKey is a serialized private key, with all parameters included + * (PRIVATE_KEY_SIZE bytes) */ typedef std::vector<unsigned char, secure_allocator<unsigned char> > CPrivKey; /** An encapsulated private key. */ class CKey { +public: + /** + * secp256k1: + */ + static const unsigned int PRIVATE_KEY_SIZE = 279; + static const unsigned int COMPRESSED_PRIVATE_KEY_SIZE = 214; + /** + * see www.keylength.com + * script supports up to 75 for single byte push + */ + static_assert( + PRIVATE_KEY_SIZE >= COMPRESSED_PRIVATE_KEY_SIZE, + "COMPRESSED_PRIVATE_KEY_SIZE is larger than PRIVATE_KEY_SIZE"); + private: //! Whether this private key is valid. We check for correctness when modifying the key //! data, so fValid should always correspond to the actual state. diff --git a/src/keystore.cpp b/src/keystore.cpp index 8454175ca8..a010a1244a 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -3,11 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "keystore.h" +#include <keystore.h> -#include "key.h" -#include "pubkey.h" -#include "util.h" +#include <util.h> bool CKeyStore::AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); @@ -36,6 +34,33 @@ bool CBasicKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) return true; } +bool CBasicKeyStore::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + return mapKeys.count(address) > 0; +} + +std::set<CKeyID> CBasicKeyStore::GetKeys() const +{ + LOCK(cs_KeyStore); + std::set<CKeyID> set_address; + for (const auto& mi : mapKeys) { + set_address.insert(mi.first); + } + return set_address; +} + +bool CBasicKeyStore::GetKey(const CKeyID &address, CKey &keyOut) const +{ + LOCK(cs_KeyStore); + KeyMap::const_iterator mi = mapKeys.find(address); + if (mi != mapKeys.end()) { + keyOut = mi->second; + return true; + } + return false; +} + bool CBasicKeyStore::AddCScript(const CScript& redeemScript) { if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) @@ -52,6 +77,16 @@ bool CBasicKeyStore::HaveCScript(const CScriptID& hash) const return mapScripts.count(hash) > 0; } +std::set<CScriptID> CBasicKeyStore::GetCScripts() const +{ + LOCK(cs_KeyStore); + std::set<CScriptID> set_script; + for (const auto& mi : mapScripts) { + set_script.insert(mi.first); + } + return set_script; +} + bool CBasicKeyStore::GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const { LOCK(cs_KeyStore); diff --git a/src/keystore.h b/src/keystore.h index 9b85ddb0ec..516a238241 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_KEYSTORE_H #define BITCOIN_KEYSTORE_H -#include "key.h" -#include "pubkey.h" -#include "script/script.h" -#include "script/standard.h" -#include "sync.h" +#include <key.h> +#include <pubkey.h> +#include <script/script.h> +#include <script/standard.h> +#include <sync.h> #include <boost/signals2/signal.hpp> @@ -36,6 +36,7 @@ public: //! Support for BIP 0013 : see https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki virtual bool AddCScript(const CScript& redeemScript) =0; virtual bool HaveCScript(const CScriptID &hash) const =0; + virtual std::set<CScriptID> GetCScripts() const =0; virtual bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const =0; //! Support for Watch-only addresses @@ -62,39 +63,12 @@ protected: public: bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - bool HaveKey(const CKeyID &address) const override - { - bool result; - { - LOCK(cs_KeyStore); - result = (mapKeys.count(address) > 0); - } - return result; - } - std::set<CKeyID> GetKeys() const override - { - LOCK(cs_KeyStore); - std::set<CKeyID> set_address; - for (const auto& mi : mapKeys) { - set_address.insert(mi.first); - } - return set_address; - } - bool GetKey(const CKeyID &address, CKey &keyOut) const override - { - { - LOCK(cs_KeyStore); - KeyMap::const_iterator mi = mapKeys.find(address); - if (mi != mapKeys.end()) - { - keyOut = mi->second; - return true; - } - } - return false; - } + bool HaveKey(const CKeyID &address) const override; + std::set<CKeyID> GetKeys() const override; + bool GetKey(const CKeyID &address, CKey &keyOut) const override; bool AddCScript(const CScript& redeemScript) override; bool HaveCScript(const CScriptID &hash) const override; + std::set<CScriptID> GetCScripts() const override; bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const override; bool AddWatchOnly(const CScript &dest) override; diff --git a/src/memusage.h b/src/memusage.h index 93fd6a0eb5..d931171d35 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_MEMUSAGE_H #define BITCOIN_MEMUSAGE_H -#include "indirectmap.h" +#include <indirectmap.h> #include <stdlib.h> diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 3f07b4dac4..f4db9dd57e 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkleblock.h" +#include <merkleblock.h> -#include "hash.h" -#include "consensus/consensus.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <consensus/consensus.h> +#include <utilstrencodings.h> CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids) diff --git a/src/merkleblock.h b/src/merkleblock.h index 6c05f2c1f8..b64c34370e 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_MERKLEBLOCK_H #define BITCOIN_MERKLEBLOCK_H -#include "serialize.h" -#include "uint256.h" -#include "primitives/block.h" -#include "bloom.h" +#include <serialize.h> +#include <uint256.h> +#include <primitives/block.h> +#include <bloom.h> #include <vector> diff --git a/src/miner.cpp b/src/miner.cpp index a9989f4b17..0fb9b774d3 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -3,29 +3,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "miner.h" - -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "consensus/tx_verify.h" -#include "consensus/merkle.h" -#include "consensus/validation.h" -#include "hash.h" -#include "validation.h" -#include "net.h" -#include "policy/feerate.h" -#include "policy/policy.h" -#include "pow.h" -#include "primitives/transaction.h" -#include "script/standard.h" -#include "timedata.h" -#include "txmempool.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validationinterface.h" +#include <miner.h> + +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <consensus/tx_verify.h> +#include <consensus/merkle.h> +#include <consensus/validation.h> +#include <hash.h> +#include <validation.h> +#include <net.h> +#include <policy/feerate.h> +#include <policy/policy.h> +#include <pow.h> +#include <primitives/transaction.h> +#include <script/standard.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validationinterface.h> #include <algorithm> #include <queue> diff --git a/src/miner.h b/src/miner.h index db165e71c6..d81ec6421c 100644 --- a/src/miner.h +++ b/src/miner.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_MINER_H #define BITCOIN_MINER_H -#include "primitives/block.h" -#include "txmempool.h" +#include <primitives/block.h> +#include <txmempool.h> #include <stdint.h> #include <memory> @@ -71,7 +71,7 @@ struct modifiedentry_iter { // except operating on CTxMemPoolModifiedEntry. // TODO: refactor to avoid duplication of this logic. struct CompareModifiedEntry { - bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b) + bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b) const { double f1 = (double)a.nModFeesWithAncestors * b.nSizeWithAncestors; double f2 = (double)b.nModFeesWithAncestors * a.nSizeWithAncestors; @@ -86,7 +86,7 @@ struct CompareModifiedEntry { // This is sufficient to sort an ancestor package in an order that is valid // to appear in a block. struct CompareTxIterByAncestorCount { - bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b) + bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b) const { if (a->GetCountWithAncestors() != b->GetCountWithAncestors()) return a->GetCountWithAncestors() < b->GetCountWithAncestors(); diff --git a/src/net.cpp b/src/net.cpp index ea3840a708..e00faaeecd 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -4,23 +4,21 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "net.h" - -#include "addrman.h" -#include "chainparams.h" -#include "clientversion.h" -#include "consensus/consensus.h" -#include "crypto/common.h" -#include "crypto/sha256.h" -#include "hash.h" -#include "primitives/transaction.h" -#include "netbase.h" -#include "scheduler.h" -#include "ui_interface.h" -#include "utilstrencodings.h" +#include <net.h> + +#include <chainparams.h> +#include <clientversion.h> +#include <consensus/consensus.h> +#include <crypto/common.h> +#include <crypto/sha256.h> +#include <primitives/transaction.h> +#include <netbase.h> +#include <scheduler.h> +#include <ui_interface.h> +#include <utilstrencodings.h> #ifdef WIN32 #include <string.h> @@ -110,13 +108,13 @@ bool GetLocal(CService& addr, const CNetAddr *paddrPeer) int nBestReachability = -1; { LOCK(cs_mapLocalHost); - for (std::map<CNetAddr, LocalServiceInfo>::iterator it = mapLocalHost.begin(); it != mapLocalHost.end(); it++) + for (const auto& entry : mapLocalHost) { - int nScore = (*it).second.nScore; - int nReachability = (*it).first.GetReachabilityFrom(paddrPeer); + int nScore = entry.second.nScore; + int nReachability = entry.first.GetReachabilityFrom(paddrPeer); if (nReachability > nBestReachability || (nReachability == nBestReachability && nScore > nBestScore)) { - addr = CService((*it).first, (*it).second.nPort); + addr = CService(entry.first, entry.second.nPort); nBestReachability = nReachability; nBestScore = nScore; } @@ -417,40 +415,48 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (addrConnect.IsValid()) { bool proxyConnectionFailed = false; - if (GetProxy(addrConnect.GetNetwork(), proxy)) + if (GetProxy(addrConnect.GetNetwork(), proxy)) { + hSocket = CreateSocket(proxy.proxy); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, &proxyConnectionFailed); - else // no proxy needed (none set for target network) + } else { + // no proxy needed (none set for target network) + hSocket = CreateSocket(addrConnect); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout); + } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to // the proxy, mark this as an attempt. addrman.Attempt(addrConnect, fCountFailure); } } else if (pszDest && GetNameProxy(proxy)) { + hSocket = CreateSocket(proxy.proxy); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, nullptr); } - if (connected) { - if (!IsSelectableSocket(hSocket)) { - LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); - CloseSocket(hSocket); - return nullptr; - } - - // Add node - NodeId id = GetNewNodeId(); - uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); - pnode->AddRef(); - - return pnode; + if (!connected) { + CloseSocket(hSocket); + return nullptr; } - return nullptr; + // Add node + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CAddress addr_bind = GetBindAddress(hSocket); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); + pnode->AddRef(); + + return pnode; } void CConnman::DumpBanlist() @@ -597,21 +603,28 @@ void CConnman::SetBanned(const banmap_t &banMap) void CConnman::SweepBanned() { int64_t now = GetTime(); - - LOCK(cs_setBanned); - banmap_t::iterator it = setBanned.begin(); - while(it != setBanned.end()) + bool notifyUI = false; { - CSubNet subNet = (*it).first; - CBanEntry banEntry = (*it).second; - if(now > banEntry.nBanUntil) + LOCK(cs_setBanned); + banmap_t::iterator it = setBanned.begin(); + while(it != setBanned.end()) { - setBanned.erase(it++); - setBannedIsDirty = true; - LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + CSubNet subNet = (*it).first; + CBanEntry banEntry = (*it).second; + if(now > banEntry.nBanUntil) + { + setBanned.erase(it++); + setBannedIsDirty = true; + notifyUI = true; + LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + } + else + ++it; } - else - ++it; + } + // update UI + if(notifyUI && clientInterface) { + clientInterface->BannedListChanged(); } } @@ -685,7 +698,7 @@ void CNode::copyStats(CNodeStats &stats) X(cleanSubVer); } X(fInbound); - X(fAddnode); + X(m_manual_connection); X(nStartingHeight); { LOCK(cs_vSend); @@ -962,6 +975,16 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction return a.nTimeConnected > b.nTimeConnected; } + +//! Sort an array by the specified comparator, then erase the last K elements. +template<typename T, typename Comparator> +static void EraseLastKElements(std::vector<T> &elements, Comparator comparator, size_t k) +{ + std::sort(elements.begin(), elements.end(), comparator); + size_t eraseSize = std::min(k, elements.size()); + elements.erase(elements.end() - eraseSize, elements.end()); +} + /** Try to find a connection to evict when the node is full. * Extreme care must be taken to avoid opening the node to attacker * triggered network partitioning. @@ -985,48 +1008,29 @@ bool CConnman::AttemptToEvictConnection() continue; NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, - (node->nServices & nRelevantServices) == nRelevantServices, + HasAllDesirableServiceFlags(node->nServices), node->fRelayTxes, node->pfilter != nullptr, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } } - if (vEvictionCandidates.empty()) return false; - // Protect connections with certain characteristics // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); // Protect the 8 nodes with the lowest minimum ping time. // An attacker cannot manipulate this metric without physically moving nodes closer to the target. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeMinPingTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(8, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); // Protect 4 nodes that most recently sent us transactions. // An attacker cannot manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeTXTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); // Protect 4 nodes that most recently sent us blocks. // An attacker cannot manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeBlockTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); // Protect the half of the remaining nodes which have been connected the longest. // This replicates the non-eviction implicit behavior, and precludes attacks that start later. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); - vEvictionCandidates.erase(vEvictionCandidates.end() - static_cast<int>(vEvictionCandidates.size() / 2), vEvictionCandidates.end()); + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeTimeConnected, vEvictionCandidates.size() / 2); if (vEvictionCandidates.empty()) return false; @@ -1037,12 +1041,12 @@ bool CConnman::AttemptToEvictConnection() int64_t nMostConnectionsTime = 0; std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; for (const NodeEvictionCandidate &node : vEvictionCandidates) { - mapNetGroupNodes[node.nKeyedNetGroup].push_back(node); - int64_t grouptime = mapNetGroupNodes[node.nKeyedNetGroup][0].nTimeConnected; - size_t groupsize = mapNetGroupNodes[node.nKeyedNetGroup].size(); + std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; + group.push_back(node); + int64_t grouptime = group[0].nTimeConnected; - if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) { - nMostConnections = groupsize; + if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { + nMostConnections = group.size(); nMostConnectionsTime = grouptime; naMostConnections = node.nKeyedNetGroup; } @@ -1112,7 +1116,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { if (IsBanned(addr) && !whitelisted) { - LogPrintf("connection from %s dropped (banned)\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); CloseSocket(hSocket); return; } @@ -1544,22 +1548,20 @@ void ThreadMapPort() void MapPort(bool fUseUPnP) { - static boost::thread* upnp_thread = nullptr; + static std::unique_ptr<boost::thread> upnp_thread; if (fUseUPnP) { if (upnp_thread) { upnp_thread->interrupt(); upnp_thread->join(); - delete upnp_thread; } - upnp_thread = new boost::thread(boost::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort)); + upnp_thread.reset(new boost::thread(boost::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort))); } else if (upnp_thread) { upnp_thread->interrupt(); upnp_thread->join(); - delete upnp_thread; - upnp_thread = nullptr; + upnp_thread.reset(); } } @@ -1602,7 +1604,7 @@ void CConnman::ThreadDNSAddressSeed() LOCK(cs_vNodes); int nRelevant = 0; for (auto pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && ((pnode->nServices & nRelevantServices) == nRelevantServices); + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; } if (nRelevant >= 2) { LogPrintf("P2P peers available. Skipped DNS seeding.\n"); @@ -1624,7 +1626,7 @@ void CConnman::ThreadDNSAddressSeed() } else { std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; - ServiceFlags requiredServiceBits = nRelevantServices; + ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = GetDNSHost(seed, &requiredServiceBits); CNetAddr resolveSource; if (!resolveSource.SetInternal(host)) { @@ -1694,6 +1696,37 @@ void CConnman::ProcessOneShot() } } +bool CConnman::GetTryNewOutboundPeer() +{ + return m_try_another_outbound_peer; +} + +void CConnman::SetTryNewOutboundPeer(bool flag) +{ + m_try_another_outbound_peer = flag; + LogPrint(BCLog::NET, "net: setting try another outbound peer=%s\n", flag ? "true" : "false"); +} + +// Return the number of peers we have over our outbound connection limit +// Exclude peers that are marked for disconnect, or are going to be +// disconnected soon (eg one-shots and feelers) +// Also exclude peers that haven't finished initial connection handshake yet +// (so that we don't decide we're over our desired connection limit, and then +// evict some peer that has finished the handshake) +int CConnman::GetExtraOutboundCount() +{ + int nOutbound = 0; + { + LOCK(cs_vNodes); + for (CNode* pnode : vNodes) { + if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + ++nOutbound; + } + } + } + return std::max(nOutbound - nMaxOutbound, 0); +} + void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { // Connect to specific addresses @@ -1705,7 +1738,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) for (const std::string& strAddr : connect) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str()); + OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), false, false, true); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -1753,17 +1786,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; - int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->fAddnode) { - - // Count the peers that have all relevant services - if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { - nOutboundRelevant++; - } + if (!pnode->fInbound && !pnode->m_manual_connection) { // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1788,7 +1815,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // * Only make a feeler connection once every few minutes. // bool fFeeler = false; - if (nOutbound >= nMaxOutbound) { + + if (nOutbound >= nMaxOutbound && !GetTryNewOutboundPeer()) { int64_t nTime = GetTimeMicros(); // The current time right now (in microseconds). if (nTime > nNextFeeler) { nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); @@ -1818,21 +1846,16 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (IsLimited(addr)) continue; - // only connect to full nodes - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) - continue; - // only consider very recently tried nodes after 30 failed attempts if (nANow - addr.nLastTry < 600 && nTries < 30) continue; - // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - ServiceFlags nRequiredServices = nRelevantServices; - if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { - nRequiredServices = REQUIRED_SERVICES; - } - - if ((addr.nServices & nRequiredServices) != nRequiredServices) { + // for non-feelers, require all the services we'll want, + // for feelers, only require they be a full node (only because most + // SPV clients don't have a good address DB available) + if (!fFeeler && !HasAllDesirableServiceFlags(addr.nServices)) { + continue; + } else if (fFeeler && !MayHaveUsefulAddressDB(addr.nServices)) { continue; } @@ -1841,13 +1864,6 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; addrConnect = addr; - - // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services - if (nOutboundRelevant >= (nMaxOutbound >> 1)) { - addrConnect.nServices = REQUIRED_SERVICES; - } else { - addrConnect.nServices = nRequiredServices; - } break; } @@ -1946,7 +1962,7 @@ void CConnman::ThreadOpenAddedConnections() } // if successful, this moves the passed grant to the constructed node -bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool fAddnode) +bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection) { // // Initiate outbound network connection @@ -1975,8 +1991,8 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai pnode->fOneShot = true; if (fFeeler) pnode->fFeeler = true; - if (fAddnode) - pnode->fAddnode = true; + if (manual_connection) + pnode->m_manual_connection = true; m_msgproc->InitializeNode(pnode); { @@ -2056,44 +2072,21 @@ bool CConnman::BindListenPort(const CService &addrBind, std::string& strError, b return false; } - SOCKET hListenSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); + SOCKET hListenSocket = CreateSocket(addrBind); if (hListenSocket == INVALID_SOCKET) { strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %s)", NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError); return false; } - if (!IsSelectableSocket(hListenSocket)) - { - strError = "Error: Couldn't create a listenable socket for incoming connections"; - LogPrintf("%s\n", strError); - return false; - } - - #ifndef WIN32 -#ifdef SO_NOSIGPIPE - // Different way of disabling SIGPIPE on BSD - setsockopt(hListenSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&nOne, sizeof(int)); -#endif // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (void*)&nOne, sizeof(int)); - // Disable Nagle's algorithm - setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&nOne, sizeof(int)); #else setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (const char*)&nOne, sizeof(int)); - setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&nOne, sizeof(int)); #endif - // Set to non-blocking, incoming connections will also inherit this - if (!SetSocketNonBlocking(hListenSocket, true)) { - CloseSocket(hListenSocket); - strError = strprintf("BindListenPort: Setting listening socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); - return false; - } - // some systems don't have IPV6_V6ONLY but are always v6only; others do have the option // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { @@ -2220,9 +2213,8 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSe nLastNodeId = 0; nSendBufferMaxSize = 0; nReceiveFloodSize = 0; - semOutbound = nullptr; - semAddnode = nullptr; flagInterruptMsgProc = false; + SetTryNewOutboundPeer(false); Options connOptions; Init(connOptions); @@ -2268,10 +2260,16 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { Init(connOptions); - nTotalBytesRecv = 0; - nTotalBytesSent = 0; - nMaxOutboundTotalBytesSentInCycle = 0; - nMaxOutboundCycleStartTime = 0; + { + LOCK(cs_totalBytesRecv); + nTotalBytesRecv = 0; + } + { + LOCK(cs_totalBytesSent); + nTotalBytesSent = 0; + nMaxOutboundTotalBytesSentInCycle = 0; + nMaxOutboundCycleStartTime = 0; + } if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds)) { if (clientInterface) { @@ -2326,11 +2324,11 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = new CSemaphore(std::min((nMaxOutbound + nMaxFeeler), nMaxConnections)); + semOutbound = MakeUnique<CSemaphore>(std::min((nMaxOutbound + nMaxFeeler), nMaxConnections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = new CSemaphore(nMaxAddnode); + semAddnode = MakeUnique<CSemaphore>(nMaxAddnode); } // @@ -2453,10 +2451,8 @@ void CConnman::Stop() vNodes.clear(); vNodesDisconnected.clear(); vhListenSocket.clear(); - delete semOutbound; - semOutbound = nullptr; - delete semAddnode; - semAddnode = nullptr; + semOutbound.reset(); + semAddnode.reset(); } void CConnman::DeleteNode(CNode* pnode) @@ -2712,7 +2708,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nSendVersion(0) { nServices = NODE_NONE; - nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -2725,7 +2720,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn strSubVer = ""; fWhitelisted = false; fOneShot = false; - fAddnode = false; + m_manual_connection = false; fClient = false; // set by version message fFeeler = false; fSuccessfullyConnected = false; @@ -2743,7 +2738,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nNextInvSend = 0; fRelayTxes = false; fSentAddr = false; - pfilter = new CBloomFilter(); + pfilter = MakeUnique<CBloomFilter>(); timeLastMempoolReq = 0; nLastBlockTime = 0; nLastTXTime = 0; @@ -2773,9 +2768,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn CNode::~CNode() { CloseSocket(hSocket); - - if (pfilter) - delete pfilter; } void CNode::AskFor(const CInv& inv) @@ -6,21 +6,21 @@ #ifndef BITCOIN_NET_H #define BITCOIN_NET_H -#include "addrdb.h" -#include "addrman.h" -#include "amount.h" -#include "bloom.h" -#include "compat.h" -#include "hash.h" -#include "limitedmap.h" -#include "netaddress.h" -#include "policy/feerate.h" -#include "protocol.h" -#include "random.h" -#include "streams.h" -#include "sync.h" -#include "uint256.h" -#include "threadinterrupt.h" +#include <addrdb.h> +#include <addrman.h> +#include <amount.h> +#include <bloom.h> +#include <compat.h> +#include <hash.h> +#include <limitedmap.h> +#include <netaddress.h> +#include <policy/feerate.h> +#include <protocol.h> +#include <random.h> +#include <streams.h> +#include <sync.h> +#include <uint256.h> +#include <threadinterrupt.h> #include <atomic> #include <deque> @@ -84,8 +84,6 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; -static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; - // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -130,7 +128,6 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - ServiceFlags nRelevantServices = NODE_NONE; int nMaxConnections = 0; int nMaxOutbound = 0; int nMaxAddnode = 0; @@ -152,7 +149,6 @@ public: void Init(const Options& connOptions) { nLocalServices = connOptions.nLocalServices; - nRelevantServices = connOptions.nRelevantServices; nMaxConnections = connOptions.nMaxConnections; nMaxOutbound = std::min(connOptions.nMaxOutbound, connOptions.nMaxConnections); nMaxAddnode = connOptions.nMaxAddnode; @@ -162,10 +158,16 @@ public: m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; nReceiveFloodSize = connOptions.nReceiveFloodSize; - nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; - nMaxOutboundLimit = connOptions.nMaxOutboundLimit; + { + LOCK(cs_totalBytesSent); + nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; + nMaxOutboundLimit = connOptions.nMaxOutboundLimit; + } vWhitelistedRange = connOptions.vWhitelistedRange; - vAddedNodes = connOptions.m_added_nodes; + { + LOCK(cs_vAddedNodes); + vAddedNodes = connOptions.m_added_nodes; + } } CConnman(uint64_t seed0, uint64_t seed1); @@ -175,7 +177,7 @@ public: void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); - bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool fAddnode = false); + bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false); bool CheckIncomingNonce(uint64_t nonce); bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); @@ -255,6 +257,19 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); + // This allows temporarily exceeding nMaxOutbound, with the goal of finding + // a peer that is better than all our current peers. + void SetTryNewOutboundPeer(bool flag); + bool GetTryNewOutboundPeer(); + + // Return the number of outbound peers we have in excess of our target (eg, + // if we previously called SetTryNewOutboundPeer(true), and have since set + // to false, we may have extra peers that we wish to disconnect). This may + // return a value less than (num_outbound_connections - num_outbound_slots) + // in cases where some outbound connections are not yet fully connected, or + // not yet fully disconnected. + int GetExtraOutboundCount(); + bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -355,14 +370,14 @@ private: // Network usage totals CCriticalSection cs_totalBytesRecv; CCriticalSection cs_totalBytesSent; - uint64_t nTotalBytesRecv; - uint64_t nTotalBytesSent; + uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv); + uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent); // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle; - uint64_t nMaxOutboundCycleStartTime; - uint64_t nMaxOutboundLimit; - uint64_t nMaxOutboundTimeframe; + uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundTimeframe GUARDED_BY(cs_totalBytesSent); // Whitelisted ranges. Any node connecting from these is automatically // whitelisted (as well as those connecting to whitelisted binds). @@ -380,7 +395,7 @@ private: CAddrMan addrman; std::deque<std::string> vOneShots; CCriticalSection cs_vOneShots; - std::vector<std::string> vAddedNodes; + std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); CCriticalSection cs_vAddedNodes; std::vector<CNode*> vNodes; std::list<CNode*> vNodesDisconnected; @@ -390,11 +405,8 @@ private: /** Services this instance offers */ ServiceFlags nLocalServices; - /** Services this instance cares about */ - ServiceFlags nRelevantServices; - - CSemaphore *semOutbound; - CSemaphore *semAddnode; + std::unique_ptr<CSemaphore> semOutbound; + std::unique_ptr<CSemaphore> semAddnode; int nMaxConnections; int nMaxOutbound; int nMaxAddnode; @@ -420,6 +432,13 @@ private: std::thread threadOpenAddedConnections; std::thread threadOpenConnections; std::thread threadMessageHandler; + + /** flag for deciding to connect to an extra outbound peer, + * in excess of nMaxOutbound + * This takes the place of a feeler connection */ + std::atomic_bool m_try_another_outbound_peer; + + friend struct CConnmanTest; }; extern std::unique_ptr<CConnman> g_connman; void Discover(boost::thread_group& threadGroup); @@ -513,7 +532,7 @@ public: int nVersion; std::string cleanSubVer; bool fInbound; - bool fAddnode; + bool m_manual_connection; int nStartingHeight; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; @@ -585,7 +604,6 @@ class CNode public: // socket std::atomic<ServiceFlags> nServices; - ServiceFlags nServicesExpected; SOCKET hSocket; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent @@ -623,7 +641,7 @@ public: bool fWhitelisted; // This peer can bypass DoS banning. bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; - bool fAddnode; + bool m_manual_connection; bool fClient; const bool fInbound; std::atomic_bool fSuccessfullyConnected; @@ -636,7 +654,7 @@ public: bool fSentAddr; CSemaphoreGrant grantOutbound; CCriticalSection cs_filter; - CBloomFilter* pfilter; + std::unique_ptr<CBloomFilter> pfilter; std::atomic<int> nRefCount; const uint64_t nKeyedNetGroup; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index b5e37722df..20426eaceb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3,33 +3,32 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "net_processing.h" - -#include "addrman.h" -#include "arith_uint256.h" -#include "blockencodings.h" -#include "chainparams.h" -#include "consensus/validation.h" -#include "hash.h" -#include "init.h" -#include "validation.h" -#include "merkleblock.h" -#include "net.h" -#include "netmessagemaker.h" -#include "netbase.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "random.h" -#include "reverse_iterator.h" -#include "tinyformat.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" -#include "validationinterface.h" +#include <net_processing.h> + +#include <addrman.h> +#include <arith_uint256.h> +#include <blockencodings.h> +#include <chainparams.h> +#include <consensus/validation.h> +#include <hash.h> +#include <init.h> +#include <validation.h> +#include <merkleblock.h> +#include <netmessagemaker.h> +#include <netbase.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <random.h> +#include <reverse_iterator.h> +#include <scheduler.h> +#include <tinyformat.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -40,7 +39,7 @@ std::atomic<int64_t> nTimeBestReceived(0); // Used only to inform the wallet of struct IteratorComparator { template<typename I> - bool operator()(const I& a, const I& b) + bool operator()(const I& a, const I& b) const { return &(*a) < &(*b); } @@ -124,6 +123,12 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; + /** Number of outbound peers with m_chain_sync.m_protect. */ + int g_outbound_peers_with_protect_from_disconnect = 0; + + /** When our tip was last updated. */ + int64_t g_last_tip_update = 0; + /** Relay map, protected by cs_main. */ typedef std::map<uint256, CTransactionRef> MapRelay; MapRelay mapRelay; @@ -201,6 +206,36 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; + /** State used to enforce CHAIN_SYNC_TIMEOUT + * Only in effect for outbound, non-manual connections, with + * m_protect == false + * Algorithm: if a peer's best known block has less work than our tip, + * set a timeout CHAIN_SYNC_TIMEOUT seconds in the future: + * - If at timeout their best known block now has more work than our tip + * when the timeout was set, then either reset the timeout or clear it + * (after comparing against our current tip's work) + * - If at timeout their best known block still has less work than our + * tip did when the timeout was set, then send a getheaders message, + * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. + * If their best known block is still behind when that new timeout is + * reached, disconnect. + */ + struct ChainSyncTimeoutState { + //! A timeout used for checking whether our peer has sufficiently synced + int64_t m_timeout; + //! A header with the work we require on our peer's chain + const CBlockIndex * m_work_header; + //! After timeout is reached, set to true after sending getheaders + bool m_sent_getheaders; + //! Whether this peer is protected from disconnection due to a bad/slow chain + bool m_protect; + }; + + ChainSyncTimeoutState m_chain_sync; + + //! Time of last new block announcement + int64_t m_last_block_announcement; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -223,6 +258,8 @@ struct CNodeState { fHaveWitness = false; fWantsCmpctWitness = false; fSupportsDesiredCmpctVersion = false; + m_chain_sync = { 0, nullptr, false, false }; + m_last_block_announcement = 0; } }; @@ -379,25 +416,32 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) { } } connman->ForNode(nodeid, [connman](CNode* pfrom){ - bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - fAnnounceUsingCMPCTBLOCK = true; - connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); } } +bool TipMayBeStale(const Consensus::Params &consensusParams) +{ + AssertLockHeld(cs_main); + if (g_last_tip_update == 0) { + g_last_tip_update = GetTime(); + } + return g_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -504,6 +548,22 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<con } // namespace +// This function is used for testing the stale tip eviction logic, see +// DoS_tests.cpp +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) +{ + LOCK(cs_main); + CNodeState *state = State(node); + if (state) state->m_last_block_announcement = time_in_seconds; +} + +// Returns true for outbound peers, excluding manual connections, feelers, and +// one-shots +bool IsOutboundDisconnectionCandidate(const CNode *node) +{ + return !(node->fInbound || node->m_manual_connection || node->fFeeler || node->fOneShot); +} + void PeerLogicValidation::InitializeNode(CNode *pnode) { CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); @@ -536,6 +596,8 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim nPreferredDownload -= state->fPreferredDownload; nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); assert(nPeersWithValidatedDownloads >= 0); + g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; + assert(g_outbound_peers_with_protect_from_disconnect >= 0); mapNodeState.erase(nodeid); @@ -544,6 +606,7 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(mapBlocksInFlight.empty()); assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); + assert(g_outbound_peers_with_protect_from_disconnect == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } @@ -568,7 +631,7 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { // mapOrphanTransactions // -void AddToCompactExtraTransactions(const CTransactionRef& tx) +void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { size_t max_extra_txn = gArgs.GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); if (max_extra_txn <= 0) @@ -716,18 +779,28 @@ void Misbehaving(NodeId pnode, int howmuch) // To prevent fingerprinting attacks, only send blocks/headers outside of the // active chain if they are no more than a month older (both in time, and in -// best equivalent proof of work) than the best header chain we know about. -static bool StaleBlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Params& consensusParams) +// best equivalent proof of work) than the best header chain we know about and +// we fully-validated them at some point. +static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Params& consensusParams) { AssertLockHeld(cs_main); - return (pindexBestHeader != nullptr) && + if (chainActive.Contains(pindex)) return true; + return pindex->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() < STALE_RELAY_AGE_LIMIT) && (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler) : connman(connmanIn), m_stale_tip_check_time(0) { // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + + const Consensus::Params& consensusParams = Params().GetConsensus(); + // Stale tip checking and peer eviction are on two different timers, but we + // don't want them to get out of sync due to drift in the scheduler, so we + // combine them in one function and schedule at the quicker (peer-eviction) + // timer. + static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); + scheduler.scheduleEvery(std::bind(&PeerLogicValidation::CheckForStaleTipAndEvictPeers, this, consensusParams), EXTRA_PEER_CHECK_INTERVAL * 1000); } void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { @@ -758,6 +831,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } + + g_last_tip_update = GetTime(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -999,14 +1074,9 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam CValidationState dummy; ActivateBestChain(dummy, Params(), a_recent_block); } - if (chainActive.Contains(mi->second)) { - send = true; - } else { - send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && - StaleBlockRequestAllowed(mi->second, consensusParams); - if (!send) { - LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); - } + send = BlockRequestAllowed(mi->second, consensusParams); + if (!send) { + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); } } // disconnect node in case we have reached the outbound limit for serving historical blocks @@ -1019,6 +1089,16 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam pfrom->fDisconnect = true; send = false; } + // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold + if (send && !pfrom->fWhitelisted && ( + (((pfrom->GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom->GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (chainActive.Tip()->nHeight - mi->second->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) + )) { + LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold from peer=%d\n", pfrom->GetId()); + + //disconnect node and prevent it from stalling (would otherwise wait for the missing block) + pfrom->fDisconnect = true; + send = false; + } // Pruned nodes may have deleted the block, so check whether // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) @@ -1166,6 +1246,225 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } +bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool punish_duplicate_invalid) +{ + const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + size_t nCount = headers.size(); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + bool received_new_header = false; + const CBlockIndex *pindexLast = nullptr; + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->GetId(), nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + hashLastBlock = header.GetHash(); + } + + // If we don't have the last header, then they'll have given us + // something new (if these headers are valid). + if (mapBlockIndex.find(hashLastBlock) == mapBlockIndex.end()) { + received_new_header = true; + } + } + + CValidationState state; + CBlockHeader first_invalid_header; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + LOCK(cs_main); + if (nDoS > 0) { + Misbehaving(pfrom->GetId(), nDoS); + } + if (punish_duplicate_invalid && mapBlockIndex.find(first_invalid_header.GetHash()) != mapBlockIndex.end()) { + // Goal: don't allow outbound peers to use up our outbound + // connection slots if they are on incompatible chains. + // + // We ask the caller to set punish_invalid appropriately based + // on the peer and the method of header delivery (compact + // blocks are allowed to be invalid in some circumstances, + // under BIP 152). + // Here, we try to detect the narrow situation that we have a + // valid block header (ie it was valid at the time the header + // was received, and hence stored in mapBlockIndex) but know the + // block is invalid, and that a peer has announced that same + // block as being on its active chain. + // Disconnect the peer in such a situation. + // + // Note: if the header that is invalid was not accepted to our + // mapBlockIndex at all, that may also be grounds for + // disconnecting the peer, as the chain they are on is likely + // to be incompatible. However, there is a circumstance where + // that does not hold: if the header's timestamp is more than + // 2 hours ahead of our current time. In that case, the header + // may become valid in the future, and we don't want to + // disconnect a peer merely for serving us one too-far-ahead + // block header, to prevent an attacker from splitting the + // network by mining a block right at the 2 hour boundary. + // + // TODO: update the DoS logic (or, rather, rewrite the + // DoS-interface between validation and net_processing) so that + // the interface is cleaner, and so that we disconnect on all the + // reasons that a peer's headers chain is incompatible + // with ours (eg block->nVersion softforks, MTP violations, + // etc), and not just the duplicate-invalid case. + pfrom->fDisconnect = true; + } + return error("invalid header received"); + } + } + + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + std::vector<const CBlockIndex*> vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { + // When nCount < MAX_HEADERS_RESULTS, we know we have no more + // headers to fetch from this peer. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if using an outbound slot (unless + // whitelisted or addnode). + // Note: We compare their tip to nMinimumChainWork (rather than + // chainActive.Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (IsOutboundDisconnectionCandidate(pfrom)) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); + pfrom->fDisconnect = true; + } + } + } + + if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { + // If this is an outbound peer, check to see if we should protect + // it from the bad/lagging chain logic. + if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom->GetId()); + nodestate->m_chain_sync.m_protect = true; + ++g_outbound_peers_with_protect_from_disconnect; + } + } + } + + return true; +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -1245,11 +1544,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { connman->SetServices(pfrom->addr, nServices); } - if (pfrom->nServicesExpected & ~nServices) + if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + strprintf("Expected to offer services %08x", GetDesirableServiceFlags(nServices)))); pfrom->fDisconnect = true; return false; } @@ -1268,7 +1567,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); + LogPrint(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1368,7 +1667,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fLogIPs) remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); - LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); @@ -1411,6 +1710,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Mark this node as currently connected, so we update its timestamp later. LOCK(cs_main); State(pfrom->GetId())->fCurrentlyConnected = true; + LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s\n", + pfrom->nVersion.load(), pfrom->nStartingHeight, pfrom->GetId(), + (fLogIPs ? strprintf(", peeraddr=%s", pfrom->addr.ToString()) : "")); } if (pfrom->nVersion >= SENDHEADERS_VERSION) { @@ -1468,7 +1770,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (interruptMsgProc) return true; - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + // We only bother storing full nodes, though this may include + // things which we would not make an outbound connection to, in + // part because we may make feeler connections to them. + if (!MayHaveUsefulAddressDB(addr.nServices)) continue; if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) @@ -1686,7 +1991,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1737,9 +2042,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; pindex = (*mi).second; - if (!chainActive.Contains(pindex) && - !StaleBlockRequestAllowed(pindex, chainparams.GetConsensus())) { - LogPrintf("%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom->GetId()); + if (!BlockRequestAllowed(pindex, chainparams.GetConsensus())) { + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom->GetId()); return true; } } @@ -1809,7 +2113,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, &fMissingInputs, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); RelayTransaction(tx, connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { vWorkQueue.emplace_back(inv.hash, i); @@ -1876,7 +2180,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(orphanHash); } } - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); } } @@ -1970,6 +2274,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; + bool received_new_header = false; + { LOCK(cs_main); @@ -1979,6 +2285,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } + + if (mapBlockIndex.find(cmpctblock.header.GetHash()) == mapBlockIndex.end()) { + received_new_header = true; + } } const CBlockIndex *pindex = nullptr; @@ -1987,10 +2297,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS; if (state.IsInvalid(nDoS)) { if (nDoS > 0) { + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); + } else { + LogPrint(BCLog::NET, "Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -2005,7 +2317,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // If we end up treating this as a plain headers message, call that as well // without cs_main. bool fRevertToHeaderProcessing = false; - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); // Keep a CBlock for "optimistic" compactblock reconstructions (see // below) @@ -2018,6 +2329,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr assert(pindex); UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this was a new header with more work than our tip, update the + // peer's last block announcement time + if (received_new_header && pindex->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); @@ -2040,8 +2359,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) return true; - CNodeState *nodestate = State(pfrom->GetId()); - if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { // Don't bother trying to process compact blocks from v1 peers // after segwit activates. @@ -2122,10 +2439,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message - // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) - std::vector<CBlock> headers; - headers.push_back(cmpctblock.header); - vHeadersMsg << headers; fRevertToHeaderProcessing = true; } } @@ -2134,8 +2447,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fProcessBLOCKTXN) return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); - if (fRevertToHeaderProcessing) - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + if (fRevertToHeaderProcessing) { + // Headers received from HB compact block peers are permitted to be + // relayed before full validation (see BIP 152), so we don't want to disconnect + // the peer if the header turns out to be for an invalid block. + // Note that if a peer tries to build on an invalid chain, that + // will be detected and the peer will be banned. + return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*punish_duplicate_invalid=*/false); + } if (fBlockReconstructed) { // If we got here, we were able to optimistically reconstruct a @@ -2145,7 +2464,16 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // Setting fForceProcessing to true means that we bypass some of + // our anti-DoS protections in AcceptBlock, which filters + // unrequested blocks that might be trying to waste our resources + // (eg disk space). Because we only try to reconstruct blocks when + // we're close to caught up (via the CanDirectFetch() requirement + // above, combined with the behavior of not requesting blocks until + // we have a chain with at least nMinimumChainWork), and we ignore + // compact blocks with less work than our tip, it is safe to treat + // reconstructed compact blocks as having been requested. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2225,7 +2553,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent + // disk-space attacks), but this should be safe due to the + // protections in the compact block handler -- see related comment + // in compact block optimistic reconstruction handling. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2253,136 +2585,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } - - const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - - // If this looks like it could be a block announcement (nCount < - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom->GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); - } - return true; - } - - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); - } - hashLastBlock = header.GetHash(); - } - } - - CValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - return error("invalid header received"); - } - } - - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector<const CBlockIndex*> vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!chainActive.Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector<CInv> vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - } + // Headers received via a HEADERS message should be valid, and reflect + // the chain the peer is on. If we receive a known-invalid header, + // disconnect the peer if it is using one of our outbound connection + // slots. + bool should_punish = !pfrom->fInbound && !pfrom->m_manual_connection; + return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing @@ -2392,11 +2600,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); - // Process all blocks from whitelisted peers, even if not requested, - // unless we're still syncing with the network. - // Such an unrequested block may still be processed, subject to the - // conditions in AcceptBlock(). - bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + bool forceProcessing = false; const uint256 hash(pblock->GetHash()); { LOCK(cs_main); @@ -2560,8 +2764,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(filter); + pfrom->pfilter.reset(new CBloomFilter(filter)); pfrom->pfilter->UpdateEmptyFull(); pfrom->fRelayTxes = true; } @@ -2597,8 +2800,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { LOCK(pfrom->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); + pfrom->pfilter.reset(new CBloomFilter()); } pfrom->fRelayTxes = true; } @@ -2644,8 +2846,8 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman* connman) state.fShouldBan = false; if (pnode->fWhitelisted) LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode->addr.ToString()); - else if (pnode->fAddnode) - LogPrintf("Warning: not punishing addnoded peer %s!\n", pnode->addr.ToString()); + else if (pnode->m_manual_connection) + LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode->addr.ToString()); else { pnode->fDisconnect = true; if (pnode->addr.IsLocal()) @@ -2702,7 +2904,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter msg.SetVersion(pfrom->GetRecvVersion()); // Scan for message start if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); pfrom->fDisconnect = true; return false; } @@ -2711,7 +2913,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter CMessageHeader& hdr = msg.hdr; if (!hdr.IsValid(chainparams.MessageStart())) { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); return fMoreWork; } std::string strCommand = hdr.GetCommand(); @@ -2724,7 +2926,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter const uint256& hash = msg.GetMessageHash(); if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, SanitizeString(strCommand), nMessageSize, HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); @@ -2747,17 +2949,17 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else { @@ -2771,7 +2973,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -2780,6 +2982,136 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return fMoreWork; } +void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) +{ + AssertLockHeld(cs_main); + + CNodeState &state = *State(pto->GetId()); + const CNetMsgMaker msgMaker(pto->GetSendVersion()); + + if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { + // This is an outbound peer subject to disconnection if they don't + // announce a block with as much work as the current tip within + // CHAIN_SYNC_TIMEOUT + HEADERS_RESPONSE_TIME seconds (note: if + // their chain has more work than ours, we should sync to it, + // unless it's invalid, in which case we should find that out and + // disconnect from them elsewhere). + if (state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork) { + if (state.m_chain_sync.m_timeout != 0) { + state.m_chain_sync.m_timeout = 0; + state.m_chain_sync.m_work_header = nullptr; + state.m_chain_sync.m_sent_getheaders = false; + } + } else if (state.m_chain_sync.m_timeout == 0 || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { + // Our best block known by this peer is behind our tip, and we're either noticing + // that for the first time, OR this peer was able to catch up to some earlier point + // where we checked against our tip. + // Either way, set a new timeout based on current tip. + state.m_chain_sync.m_timeout = time_in_seconds + CHAIN_SYNC_TIMEOUT; + state.m_chain_sync.m_work_header = chainActive.Tip(); + state.m_chain_sync.m_sent_getheaders = false; + } else if (state.m_chain_sync.m_timeout > 0 && time_in_seconds > state.m_chain_sync.m_timeout) { + // No evidence yet that our peer has synced to a chain with work equal to that + // of our tip, when we first detected it was behind. Send a single getheaders + // message to give the peer a chance to update us. + if (state.m_chain_sync.m_sent_getheaders) { + // They've run out of time to catch up! + LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); + pto->fDisconnect = true; + } else { + assert(state.m_chain_sync.m_work_header); + LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + state.m_chain_sync.m_sent_getheaders = true; + constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes + // Bump the timeout to allow a response, which could clear the timeout + // (if the response shows the peer has synced), reset the timeout (if + // the peer syncs to the required work but not to our tip), or result + // in disconnect (if we advance to the timeout and pindexBestKnownBlock + // has not sufficiently progressed) + state.m_chain_sync.m_timeout = time_in_seconds + HEADERS_RESPONSE_TIME; + } + } + } +} + +void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) +{ + // Check whether we have too many outbound peers + int extra_peers = connman->GetExtraOutboundCount(); + if (extra_peers > 0) { + // If we have more outbound peers than we target, disconnect one. + // Pick the outbound peer that least recently announced + // us a new block, with ties broken by choosing the more recent + // connection (higher node id) + NodeId worst_peer = -1; + int64_t oldest_block_announcement = std::numeric_limits<int64_t>::max(); + + LOCK(cs_main); + + connman->ForEachNode([&](CNode* pnode) { + // Ignore non-outbound peers, or nodes marked for disconnect already + if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; + CNodeState *state = State(pnode->GetId()); + if (state == nullptr) return; // shouldn't be possible, but just in case + // Don't evict our protected peers + if (state->m_chain_sync.m_protect) return; + if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { + worst_peer = pnode->GetId(); + oldest_block_announcement = state->m_last_block_announcement; + } + }); + if (worst_peer != -1) { + bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { + // Only disconnect a peer that has been connected to us for + // some reasonable fraction of our check-frequency, to give + // it time for new information to have arrived. + // Also don't disconnect any peer we're trying to download a + // block from. + CNodeState &state = *State(pnode->GetId()); + if (time_in_seconds - pnode->nTimeConnected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); + pnode->fDisconnect = true; + return true; + } else { + LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", pnode->GetId(), pnode->nTimeConnected, state.nBlocksInFlight); + return false; + } + }); + if (disconnected) { + // If we disconnected an extra peer, that means we successfully + // connected to at least one peer after the last time we + // detected a stale tip. Don't try any more extra peers until + // we next detect a stale tip, to limit the load we put on the + // network from these extra connections. + connman->SetTryNewOutboundPeer(false); + } + } + } +} + +void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) +{ + if (connman == nullptr) return; + + int64_t time_in_seconds = GetTime(); + + EvictExtraOutboundPeers(time_in_seconds); + + if (time_in_seconds > m_stale_tip_check_time) { + LOCK(cs_main); + // Check whether our tip is stale, and if so, allow using an extra + // outbound peer + if (TipMayBeStale(consensusParams)) { + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); + connman->SetTryNewOutboundPeer(true); + } else if (connman->GetTryNewOutboundPeer()) { + connman->SetTryNewOutboundPeer(false); + } + m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; + } +} + class CompareInvMempoolOrder { CTxMemPool *mp; @@ -3246,6 +3578,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic<bool>& interruptM } } + // Check that outbound peers have reasonable chains + // GetTime() is used by this anti-DoS logic so we can test this using mocktime + ConsiderEviction(pto, GetTime()); // // Message: getdata (blocks) diff --git a/src/net_processing.h b/src/net_processing.h index 79745cdd42..3fc1e05835 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -6,8 +6,9 @@ #ifndef BITCOIN_NET_PROCESSING_H #define BITCOIN_NET_PROCESSING_H -#include "net.h" -#include "validationinterface.h" +#include <net.h> +#include <validationinterface.h> +#include <consensus/params.h> /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -21,13 +22,25 @@ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Protect at least this many outbound peers from disconnection due to slow/ + * behind headers chain. + */ +static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; +/** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ +static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes +/** How frequently to check for stale tips, in seconds */ +static constexpr int64_t STALE_CHECK_INTERVAL = 10 * 60; // 10 minutes +/** How frequently to check for extra outbound peers and disconnect, in seconds */ +static constexpr int64_t EXTRA_PEER_CHECK_INTERVAL = 45; +/** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict, in seconds */ +static constexpr int64_t MINIMUM_CONNECT_TIME = 30; class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: - CConnman* connman; + CConnman* const connman; public: - explicit PeerLogicValidation(CConnman* connman); + explicit PeerLogicValidation(CConnman* connman, CScheduler &scheduler); void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; @@ -47,6 +60,13 @@ public: * @return True if there is more work to be done */ bool SendMessages(CNode* pto, std::atomic<bool>& interrupt) override; + + void ConsiderEviction(CNode *pto, int64_t time_in_seconds); + void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); + void EvictExtraOutboundPeers(int64_t time_in_seconds); + +private: + int64_t m_stale_tip_check_time; //! Next time to check for stale tip }; struct CNodeStateStats { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index b8a261c921..fa1c191559 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,14 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifdef HAVE_CONFIG_H -#include "config/bitcoin-config.h" -#endif - -#include "netaddress.h" -#include "hash.h" -#include "utilstrencodings.h" -#include "tinyformat.h" +#include <netaddress.h> +#include <hash.h> +#include <utilstrencodings.h> +#include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; diff --git a/src/netaddress.h b/src/netaddress.h index 6ca99b36b5..3b542b4748 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -6,11 +6,11 @@ #define BITCOIN_NETADDRESS_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "serialize.h" +#include <compat.h> +#include <serialize.h> #include <stdint.h> #include <string> diff --git a/src/netbase.cpp b/src/netbase.cpp index 5a560bc95a..f4232fc42a 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -3,18 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifdef HAVE_CONFIG_H -#include "config/bitcoin-config.h" -#endif - -#include "netbase.h" +#include <netbase.h> -#include "hash.h" -#include "sync.h" -#include "uint256.h" -#include "random.h" -#include "util.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <sync.h> +#include <uint256.h> +#include <random.h> +#include <util.h> +#include <utilstrencodings.h> #include <atomic> @@ -291,7 +287,7 @@ struct ProxyCredentials std::string password; }; -/** Convert SOCKS5 reply to a an error message */ +/** Convert SOCKS5 reply to an error message */ std::string Socks5ErrorString(uint8_t err) { switch(err) { @@ -317,12 +313,11 @@ std::string Socks5ErrorString(uint8_t err) } /** Connect using SOCKS5 (as described in RFC1928) */ -static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) +static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, const SOCKET& hSocket) { IntrRecvError recvr; LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { - CloseSocket(hSocket); return error("Hostname too long"); } // Accepted authentication methods @@ -338,17 +333,14 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials } ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { - CloseSocket(hSocket); return error("Error sending to proxy"); } uint8_t pchRet1[2]; if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; } if (pchRet1[0] != SOCKSVersion::SOCKS5) { - CloseSocket(hSocket); return error("Proxy failed to initialize"); } if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { @@ -363,23 +355,19 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { - CloseSocket(hSocket); return error("Error sending authentication to proxy"); } LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); uint8_t pchRetA[2]; if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading proxy authentication response"); } if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - CloseSocket(hSocket); return error("Proxy authentication unsuccessful"); } } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { // Perform no authentication } else { - CloseSocket(hSocket); return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } std::vector<uint8_t> vSocks5; @@ -393,12 +381,10 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.push_back((port >> 0) & 0xFF); ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { - CloseSocket(hSocket); return error("Error sending to proxy"); } uint8_t pchRet2[4]; if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); if (recvr == IntrRecvError::Timeout) { /* If a timeout happens here, this effectively means we timed out while connecting * to the remote node. This is very common for Tor, so do not print an @@ -409,17 +395,14 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials } } if (pchRet2[0] != SOCKSVersion::SOCKS5) { - CloseSocket(hSocket); return error("Proxy failed to accept request"); } if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { // Failures to connect to a peer that are not proxy errors - CloseSocket(hSocket); LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); return false; } if (pchRet2[2] != 0x00) { // Reserved field must be 0 - CloseSocket(hSocket); return error("Error: malformed proxy response"); } uint8_t pchRet3[256]; @@ -431,41 +414,42 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials { recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, hSocket); if (recvr != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } int nRecv = pchRet3[0]; recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, hSocket); break; } - default: CloseSocket(hSocket); return error("Error: malformed proxy response"); + default: return error("Error: malformed proxy response"); } if (recvr != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } -bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int nTimeout) +SOCKET CreateSocket(const CService &addrConnect) { - hSocketRet = INVALID_SOCKET; - struct sockaddr_storage sockaddr; socklen_t len = sizeof(sockaddr); if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot connect to %s: unsupported network\n", addrConnect.ToString()); - return false; + LogPrintf("Cannot create socket for %s: unsupported network\n", addrConnect.ToString()); + return INVALID_SOCKET; } SOCKET hSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); if (hSocket == INVALID_SOCKET) - return false; + return INVALID_SOCKET; + + if (!IsSelectableSocket(hSocket)) { + CloseSocket(hSocket); + LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); + return INVALID_SOCKET; + } #ifdef SO_NOSIGPIPE int set = 1; @@ -479,9 +463,23 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) { CloseSocket(hSocket); - return error("ConnectSocketDirectly: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); + LogPrintf("ConnectSocketDirectly: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); } + return hSocket; +} +bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocket, int nTimeout) +{ + struct sockaddr_storage sockaddr; + socklen_t len = sizeof(sockaddr); + if (hSocket == INVALID_SOCKET) { + LogPrintf("Cannot connect to %s: invalid socket\n", addrConnect.ToString()); + return false; + } + if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { + LogPrintf("Cannot connect to %s: unsupported network\n", addrConnect.ToString()); + return false; + } if (connect(hSocket, (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); @@ -496,13 +494,11 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int if (nRet == 0) { LogPrint(BCLog::NET, "connection to %s timeout\n", addrConnect.ToString()); - CloseSocket(hSocket); return false; } if (nRet == SOCKET_ERROR) { LogPrintf("select() for %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } socklen_t nRetSize = sizeof(nRet); @@ -513,13 +509,11 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int #endif { LogPrintf("getsockopt() for %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } if (nRet != 0) { LogPrintf("connect() to %s failed after select(): %s\n", addrConnect.ToString(), NetworkErrorString(nRet)); - CloseSocket(hSocket); return false; } } @@ -530,12 +524,9 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int #endif { LogPrintf("connect() to %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } } - - hSocketRet = hSocket; return true; } @@ -587,9 +578,8 @@ bool IsProxy(const CNetAddr &addr) { return false; } -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool *outProxyConnectionFailed) { - SOCKET hSocket = INVALID_SOCKET; // first connect to proxy server if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout)) { if (outProxyConnectionFailed) @@ -601,14 +591,14 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) + if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) { return false; + } } else { - if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) + if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) { return false; + } } - - hSocketRet = hSocket; return true; } bool LookupSubNet(const char* pszName, CSubNet& ret) diff --git a/src/netbase.h b/src/netbase.h index e7d7bcb375..c89bd5b88f 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -6,12 +6,12 @@ #define BITCOIN_NETBASE_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "netaddress.h" -#include "serialize.h" +#include <compat.h> +#include <netaddress.h> +#include <serialize.h> #include <stdint.h> #include <string> @@ -51,8 +51,9 @@ bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLoo bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); CService LookupNumeric(const char *pszName, int portDefault = 0); bool LookupSubNet(const char *pszName, CSubNet& subnet); -bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int nTimeout); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); +SOCKET CreateSocket(const CService &addrConnect); +bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout); +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); /** Close socket and set hSocket to INVALID_SOCKET */ diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index 79b2501c5d..ea7fccacb2 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_NETMESSAGEMAKER_H #define BITCOIN_NETMESSAGEMAKER_H -#include "net.h" -#include "serialize.h" +#include <net.h> +#include <serialize.h> class CNetMsgMaker { diff --git a/src/noui.cpp b/src/noui.cpp index 4bfb75fa29..feb3464a56 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -3,10 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "noui.h" +#include <noui.h> -#include "ui_interface.h" -#include "util.h" +#include <ui_interface.h> +#include <util.h> #include <cstdio> #include <stdint.h> diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index a089c02284..26c44ee0cf 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -3,9 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "feerate.h" +#include <policy/feerate.h> -#include "tinyformat.h" +#include <tinyformat.h> const std::string CURRENCY_UNIT = "BTC"; diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 3449cdd699..35b1b23786 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_POLICY_FEERATE_H #define BITCOIN_POLICY_FEERATE_H -#include "amount.h" -#include "serialize.h" +#include <amount.h> +#include <serialize.h> #include <string> diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 8056f385ab..490986fc11 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -3,16 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/fees.h" -#include "policy/policy.h" +#include <policy/fees.h> +#include <policy/policy.h> -#include "amount.h" -#include "clientversion.h" -#include "primitives/transaction.h" -#include "random.h" -#include "streams.h" -#include "txmempool.h" -#include "util.h" +#include <clientversion.h> +#include <primitives/transaction.h> +#include <streams.h> +#include <txmempool.h> +#include <util.h> static constexpr double INF_FEERATE = 1e99; @@ -180,6 +178,7 @@ TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; + assert(_scale != 0 && "_scale must be non-zero"); scale = _scale; confAvg.resize(maxPeriods); for (unsigned int i = 0; i < maxPeriods; i++) { @@ -412,12 +411,13 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets size_t maxConfirms, maxPeriods; // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor - if (nFileVersion >= 149900) { - filein >> decay; - if (decay <= 0 || decay >= 1) { - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - } - filein >> scale; + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + if (scale == 0) { + throw std::runtime_error("Corrupt estimates file. Scale must be non-zero"); } filein >> avg; @@ -441,20 +441,13 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets } } - if (nFileVersion >= 149900) { - filein >> failAvg; - if (maxPeriods != failAvg.size()) { - throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); - } - for (unsigned int i = 0; i < maxPeriods; i++) { - if (failAvg[i].size() != numBuckets) { - throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); - } - } - } else { - failAvg.resize(confAvg.size()); - for (unsigned int i = 0; i < failAvg.size(); i++) { - failAvg[i].resize(numBuckets); + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); } } @@ -503,6 +496,7 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe } } if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + assert(scale != 0); unsigned int periodsAgo = blocksAgo / scale; for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { failAvg[i][bucketindex]++; @@ -543,16 +537,13 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() bucketMap[INF_FEERATE] = bucketIndex; assert(bucketMap.size() == buckets.size()); - feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); - shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); - longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); + feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); } CBlockPolicyEstimator::~CBlockPolicyEstimator() { - delete feeStats; - delete shortStats; - delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) @@ -563,7 +554,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo if (mapMemPoolTxs.count(hash)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); - return; + return; } if (txHeight != nBestSeenHeight) { @@ -685,16 +676,16 @@ CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThr double sufficientTxs = SUFFICIENT_FEETXS; switch (horizon) { case FeeEstimateHorizon::SHORT_HALFLIFE: { - stats = shortStats; + stats = shortStats.get(); sufficientTxs = SUFFICIENT_TXS_SHORT; break; } case FeeEstimateHorizon::MED_HALFLIFE: { - stats = feeStats; + stats = feeStats.get(); break; } case FeeEstimateHorizon::LONG_HALFLIFE: { - stats = longStats; + stats = longStats.get(); break; } default: { @@ -944,32 +935,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) unsigned int nFileBestSeenHeight; filein >> nFileBestSeenHeight; - if (nVersionThatWrote < 149900) { - // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. - // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor - double tempDecay; - filein >> tempDecay; - if (tempDecay <= 0 || tempDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - - std::vector<double> tempBuckets; - filein >> tempBuckets; - size_t tempNum = tempBuckets.size(); - if (tempNum <= 1 || tempNum > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - - std::map<double, unsigned int> tempMap; - - std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); - tempFeeStats->Read(filein, nVersionThatWrote, tempNum); - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. - - tempMap.clear(); - for (unsigned int i = 0; i < tempBuckets.size(); i++) { - tempMap[tempBuckets[i]] = i; - } - } - else { // nVersionThatWrote >= 149900 + if (nVersionRequired < 149900) { + LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired); + } else { // New format introduced in 149900 unsigned int nFileHistoricalFirst, nFileHistoricalBest; filein >> nFileHistoricalFirst >> nFileHistoricalBest; if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { @@ -997,12 +965,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) } // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap - delete feeStats; - delete shortStats; - delete longStats; - feeStats = fileFeeStats.release(); - shortStats = fileShortStats.release(); - longStats = fileLongStats.release(); + feeStats = std::move(fileFeeStats); + shortStats = std::move(fileShortStats); + longStats = std::move(fileLongStats); nBestSeenHeight = nFileBestSeenHeight; historicalFirst = nFileHistoricalFirst; diff --git a/src/policy/fees.h b/src/policy/fees.h index 6edaf28714..6528560f52 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -5,11 +5,11 @@ #ifndef BITCOIN_POLICYESTIMATOR_H #define BITCOIN_POLICYESTIMATOR_H -#include "amount.h" -#include "feerate.h" -#include "uint256.h" -#include "random.h" -#include "sync.h" +#include <amount.h> +#include <policy/feerate.h> +#include <uint256.h> +#include <random.h> +#include <sync.h> #include <map> #include <string> @@ -245,9 +245,9 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats* feeStats; - TxConfirmStats* shortStats; - TxConfirmStats* longStats; + std::unique_ptr<TxConfirmStats> feeStats; + std::unique_ptr<TxConfirmStats> shortStats; + std::unique_ptr<TxConfirmStats> longStats; unsigned int trackedTxs; unsigned int untrackedTxs; diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index b2fb284508..b69bed5d6f 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -5,14 +5,14 @@ // NOTE: This file is intended to be customised by the end user, and includes only local node policy logic -#include "policy/policy.h" - -#include "consensus/validation.h" -#include "validation.h" -#include "coins.h" -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" +#include <policy/policy.h> + +#include <consensus/validation.h> +#include <validation.h> +#include <coins.h> +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) diff --git a/src/policy/policy.h b/src/policy/policy.h index ef71dd73bc..f3f8ebbbb4 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_POLICY_POLICY_H #define BITCOIN_POLICY_POLICY_H -#include "consensus/consensus.h" -#include "feerate.h" -#include "script/interpreter.h" -#include "script/standard.h" +#include <consensus/consensus.h> +#include <policy/feerate.h> +#include <script/interpreter.h> +#include <script/standard.h> #include <string> diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 755ef83c9a..e9692d4b48 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/rbf.h" +#include <policy/rbf.h> bool SignalsOptInRBF(const CTransaction &tx) { diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 22c73f3319..1a5218e120 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_POLICY_RBF_H #define BITCOIN_POLICY_RBF_H -#include "txmempool.h" +#include <txmempool.h> static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; diff --git a/src/pow.cpp b/src/pow.cpp index 7d87c6bbb7..cecb54ca7b 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "pow.h" +#include <pow.h> -#include "arith_uint256.h" -#include "chain.h" -#include "primitives/block.h" -#include "uint256.h" +#include <arith_uint256.h> +#include <chain.h> +#include <primitives/block.h> +#include <uint256.h> unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHeader *pblock, const Consensus::Params& params) { @@ -6,7 +6,7 @@ #ifndef BITCOIN_POW_H #define BITCOIN_POW_H -#include "consensus/params.h" +#include <consensus/params.h> #include <stdint.h> diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 3774ac3e4b..9e20565511 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "primitives/block.h" +#include <primitives/block.h> -#include "hash.h" -#include "tinyformat.h" -#include "utilstrencodings.h" -#include "crypto/common.h" +#include <hash.h> +#include <tinyformat.h> +#include <utilstrencodings.h> +#include <crypto/common.h> uint256 CBlockHeader::GetHash() const { diff --git a/src/primitives/block.h b/src/primitives/block.h index 292df40896..4957958c6a 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_PRIMITIVES_BLOCK_H #define BITCOIN_PRIMITIVES_BLOCK_H -#include "primitives/transaction.h" -#include "serialize.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <serialize.h> +#include <uint256.h> /** Nodes collect new transactions into a block, hash them into a hash tree, * and scan through nonce values to make the block's hash satisfy proof-of-work diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index e0a106adb9..d8230d1423 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "primitives/transaction.h" +#include <primitives/transaction.h> -#include "hash.h" -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <tinyformat.h> +#include <utilstrencodings.h> std::string COutPoint::ToString() const { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 18d524e23d..32f45e5748 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,10 +7,10 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include <stdint.h> -#include "amount.h" -#include "script/script.h" -#include "serialize.h" -#include "uint256.h" +#include <amount.h> +#include <script/script.h> +#include <serialize.h> +#include <uint256.h> static const int SERIALIZE_TRANSACTION_NO_WITNESS = 0x40000000; diff --git a/src/protocol.cpp b/src/protocol.cpp index da87e40091..8c34c4196e 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -3,10 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "protocol.h" +#include <protocol.h> -#include "util.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilstrencodings.h> #ifndef WIN32 # include <arpa/inet.h> diff --git a/src/protocol.h b/src/protocol.h index 67e01d9606..cf1d40db77 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -10,10 +10,10 @@ #ifndef BITCOIN_PROTOCOL_H #define BITCOIN_PROTOCOL_H -#include "netaddress.h" -#include "serialize.h" -#include "uint256.h" -#include "version.h" +#include <netaddress.h> +#include <serialize.h> +#include <uint256.h> +#include <version.h> #include <stdint.h> #include <string> @@ -27,16 +27,13 @@ class CMessageHeader { public: - enum { - MESSAGE_START_SIZE = 4, - COMMAND_SIZE = 12, - MESSAGE_SIZE_SIZE = 4, - CHECKSUM_SIZE = 4, - - MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE, - CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE, - HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE - }; + static constexpr size_t MESSAGE_START_SIZE = 4; + static constexpr size_t COMMAND_SIZE = 12; + static constexpr size_t MESSAGE_SIZE_SIZE = 4; + static constexpr size_t CHECKSUM_SIZE = 4; + static constexpr size_t MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE; + static constexpr size_t CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE; + static constexpr size_t HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE; typedef unsigned char MessageStartChars[MESSAGE_START_SIZE]; explicit CMessageHeader(const MessageStartChars& pchMessageStartIn); @@ -249,9 +246,8 @@ const std::vector<std::string> &getAllNetMessageTypes(); enum ServiceFlags : uint64_t { // Nothing NODE_NONE = 0, - // NODE_NETWORK means that the node is capable of serving the block chain. It is currently - // set by all Bitcoin Core nodes, and is unset by SPV clients or other peers that just want - // network services but don't provide them. + // NODE_NETWORK means that the node is capable of serving the complete block chain. It is currently + // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), // NODE_GETUTXO means the node is capable of responding to the getutxo protocol request. // Bitcoin Core does not support this but a patch set called Bitcoin XT does. @@ -267,6 +263,10 @@ enum ServiceFlags : uint64_t { // NODE_XTHIN means the node supports Xtreme Thinblocks // If this is turned off then the node will not service nor make xthin requests NODE_XTHIN = (1 << 4), + // NODE_NETWORK_LIMITED means the same as NODE_NETWORK with the limitation of only + // serving the last 288 (2 day) blocks + // See BIP159 for details on how this is implemented. + NODE_NETWORK_LIMITED = (1 << 10), // Bits 24-31 are reserved for temporary experiments. Just pick a bit that // isn't getting used, or one not being used much, and notify the @@ -277,6 +277,43 @@ enum ServiceFlags : uint64_t { // BIP process. }; +/** + * Gets the set of service flags which are "desirable" for a given peer. + * + * These are the flags which are required for a peer to support for them + * to be "interesting" to us, ie for us to wish to use one of our few + * outbound connection slots for or for us to wish to prioritize keeping + * their connection around. + * + * Relevant service flags may be peer- and state-specific in that the + * version of the peer may determine which flags are required (eg in the + * case of NODE_NETWORK_LIMITED where we seek out NODE_NETWORK peers + * unless they set NODE_NETWORK_LIMITED and we are out of IBD, in which + * case NODE_NETWORK_LIMITED suffices). + * + * Thus, generally, avoid calling with peerServices == NODE_NONE. + */ +static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { + return ServiceFlags(NODE_NETWORK | NODE_WITNESS); +} + +/** + * A shortcut for (services & GetDesirableServiceFlags(services)) + * == GetDesirableServiceFlags(services), ie determines whether the given + * set of service flags are sufficient for a peer to be "relevant". + */ +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { + return !(GetDesirableServiceFlags(services) & (~services)); +} + +/** + * Checks if a peer with the given service flags may be capable of having a + * robust address-storage DB. Currently an alias for checking NODE_NETWORK. + */ +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { + return services & NODE_NETWORK; +} + /** A CService with information about it as peer */ class CAddress : public CService { diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 2da7be783f..e52acf077c 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -1,8 +1,9 @@ // Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "pubkey.h" +#include <pubkey.h> #include <secp256k1.h> #include <secp256k1_recovery.h> @@ -46,7 +47,7 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } pos += lenbyte; @@ -65,14 +66,15 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { pos++; lenbyte--; } - if (lenbyte >= sizeof(size_t)) { + static_assert(sizeof(size_t) >= 4, "size_t too small"); + if (lenbyte >= 4) { return 0; } rlen = 0; @@ -103,14 +105,15 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { pos++; lenbyte--; } - if (lenbyte >= sizeof(size_t)) { + static_assert(sizeof(size_t) >= 4, "size_t too small"); + if (lenbyte >= 4) { return 0; } slen = 0; @@ -126,7 +129,6 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 return 0; } spos = pos; - pos += slen; /* Ignore leading zeroes in R */ while (rlen > 0 && input[rpos] == 0) { @@ -182,7 +184,7 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS } bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned char>& vchSig) { - if (vchSig.size() != 65) + if (vchSig.size() != COMPACT_SIGNATURE_SIZE) return false; int recid = (vchSig[0] - 27) & 3; bool fComp = ((vchSig[0] - 27) & 4) != 0; @@ -194,8 +196,8 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha if (!secp256k1_ecdsa_recover(secp256k1_context_verify, &pubkey, &sig, hash.begin())) { return false; } - unsigned char pub[65]; - size_t publen = 65; + unsigned char pub[PUBLIC_KEY_SIZE]; + size_t publen = PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -215,8 +217,8 @@ bool CPubKey::Decompress() { if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { return false; } - unsigned char pub[65]; - size_t publen = 65; + unsigned char pub[PUBLIC_KEY_SIZE]; + size_t publen = PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -225,7 +227,7 @@ bool CPubKey::Decompress() { bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert((nChild >> 31) == 0); - assert(begin() + 33 == end()); + assert(size() == COMPRESSED_PUBLIC_KEY_SIZE); unsigned char out[64]; BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); @@ -236,8 +238,8 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { return false; } - unsigned char pub[33]; - size_t publen = 33; + unsigned char pub[COMPRESSED_PUBLIC_KEY_SIZE]; + size_t publen = COMPRESSED_PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); pubkeyChild.Set(pub, pub + publen); return true; @@ -249,8 +251,8 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; memcpy(code+9, chaincode.begin(), 32); - assert(pubkey.size() == 33); - memcpy(code+41, pubkey.begin(), 33); + assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); } void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { diff --git a/src/pubkey.h b/src/pubkey.h index 65738d8fe2..e6c32913de 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -1,28 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_PUBKEY_H #define BITCOIN_PUBKEY_H -#include "hash.h" -#include "serialize.h" -#include "uint256.h" +#include <hash.h> +#include <serialize.h> +#include <uint256.h> #include <stdexcept> #include <vector> -/** - * secp256k1: - * const unsigned int PRIVATE_KEY_SIZE = 279; - * const unsigned int PUBLIC_KEY_SIZE = 65; - * const unsigned int SIGNATURE_SIZE = 72; - * - * see www.keylength.com - * script supports up to 75 for single byte push - */ - const unsigned int BIP32_EXTKEY_SIZE = 74; /** A reference to a CKey: the Hash160 of its serialized public key */ @@ -38,21 +29,37 @@ typedef uint256 ChainCode; /** An encapsulated public key. */ class CPubKey { +public: + /** + * secp256k1: + */ + static const unsigned int PUBLIC_KEY_SIZE = 65; + static const unsigned int COMPRESSED_PUBLIC_KEY_SIZE = 33; + static const unsigned int SIGNATURE_SIZE = 72; + static const unsigned int COMPACT_SIGNATURE_SIZE = 65; + /** + * see www.keylength.com + * script supports up to 75 for single byte push + */ + static_assert( + PUBLIC_KEY_SIZE >= COMPRESSED_PUBLIC_KEY_SIZE, + "COMPRESSED_PUBLIC_KEY_SIZE is larger than PUBLIC_KEY_SIZE"); + private: /** * Just store the serialized data. * Its length can very cheaply be computed from the first byte. */ - unsigned char vch[65]; + unsigned char vch[PUBLIC_KEY_SIZE]; //! Compute the length of a pubkey with a given first byte. unsigned int static GetLen(unsigned char chHeader) { if (chHeader == 2 || chHeader == 3) - return 33; + return COMPRESSED_PUBLIC_KEY_SIZE; if (chHeader == 4 || chHeader == 6 || chHeader == 7) - return 65; + return PUBLIC_KEY_SIZE; return 0; } @@ -127,7 +134,7 @@ public: void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); - if (len <= 65) { + if (len <= PUBLIC_KEY_SIZE) { s.read((char*)vch, len); } else { // invalid pubkey, skip available data @@ -166,7 +173,7 @@ public: //! Check whether this is a compressed public key. bool IsCompressed() const { - return size() == 33; + return size() == COMPRESSED_PUBLIC_KEY_SIZE; } /** diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index f295bd4689..dda8dce391 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -3,18 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "addressbookpage.h" -#include "ui_addressbookpage.h" +#include <qt/addressbookpage.h> +#include <qt/forms/ui_addressbookpage.h> -#include "addresstablemodel.h" -#include "bitcoingui.h" -#include "csvmodelwriter.h" -#include "editaddressdialog.h" -#include "guiutil.h" -#include "platformstyle.h" +#include <qt/addresstablemodel.h> +#include <qt/bitcoingui.h> +#include <qt/csvmodelwriter.h> +#include <qt/editaddressdialog.h> +#include <qt/guiutil.h> +#include <qt/platformstyle.h> #include <QIcon> #include <QMenu> diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 0eb7ec4306..1d16940acb 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addresstablemodel.h" +#include <qt/addresstablemodel.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> -#include "base58.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <wallet/wallet.h> #include <QFont> diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index e9f5c77a5b..a720ac956b 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -3,16 +3,16 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "askpassphrasedialog.h" -#include "ui_askpassphrasedialog.h" +#include <qt/askpassphrasedialog.h> +#include <qt/forms/ui_askpassphrasedialog.h> -#include "guiconstants.h" -#include "walletmodel.h" +#include <qt/guiconstants.h> +#include <qt/walletmodel.h> -#include "support/allocators/secure.h" +#include <support/allocators/secure.h> #include <QKeyEvent> #include <QMessageBox> @@ -70,6 +70,7 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent) : break; } textChanged(); + connect(ui->toggleShowPasswordButton, SIGNAL(toggled(bool)), this, SLOT(toggleShowPassword(bool))); connect(ui->passEdit1, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); connect(ui->passEdit2, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); connect(ui->passEdit3, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); @@ -234,6 +235,15 @@ bool AskPassphraseDialog::event(QEvent *event) return QWidget::event(event); } +void AskPassphraseDialog::toggleShowPassword(bool show) +{ + ui->toggleShowPasswordButton->setDown(show); + const auto mode = show ? QLineEdit::Normal : QLineEdit::Password; + ui->passEdit1->setEchoMode(mode); + ui->passEdit2->setEchoMode(mode); + ui->passEdit3->setEchoMode(mode); +} + bool AskPassphraseDialog::eventFilter(QObject *object, QEvent *event) { /* Detect Caps Lock. diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 34bf7ccb31..7c6acc4650 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -43,6 +43,7 @@ private: private Q_SLOTS: void textChanged(); void secureClearPassFields(); + void toggleShowPassword(bool); protected: bool event(QEvent *event); diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index f8a99506c1..c96bdfd5d7 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bantablemodel.h" +#include <qt/bantablemodel.h> -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> -#include "sync.h" -#include "utiltime.h" +#include <sync.h> +#include <utiltime.h> #include <QDebug> #include <QList> @@ -55,11 +55,11 @@ public: #if QT_VERSION >= 0x040700 cachedBanlist.reserve(banMap.size()); #endif - for (banmap_t::iterator it = banMap.begin(); it != banMap.end(); it++) + for (const auto& entry : banMap) { CCombinedBan banEntry; - banEntry.subnet = (*it).first; - banEntry.banEntry = (*it).second; + banEntry.subnet = entry.first; + banEntry.banEntry = entry.second; cachedBanlist.append(banEntry); } diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 062cfdc931..dc69dee20a 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BANTABLEMODEL_H #define BITCOIN_QT_BANTABLEMODEL_H -#include "net.h" +#include <net.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 3fd58a2f9a..557d7efcec 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -3,38 +3,38 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "bitcoingui.h" - -#include "chainparams.h" -#include "clientmodel.h" -#include "fs.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "intro.h" -#include "networkstyle.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "splashscreen.h" -#include "utilitydialog.h" -#include "winshutdownmonitor.h" +#include <qt/bitcoingui.h> + +#include <chainparams.h> +#include <qt/clientmodel.h> +#include <fs.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/intro.h> +#include <qt/networkstyle.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/splashscreen.h> +#include <qt/utilitydialog.h> +#include <qt/winshutdownmonitor.h> #ifdef ENABLE_WALLET -#include "paymentserver.h" -#include "walletmodel.h" +#include <qt/paymentserver.h> +#include <qt/walletmodel.h> #endif -#include "init.h" -#include "rpc/server.h" -#include "scheduler.h" -#include "ui_interface.h" -#include "util.h" -#include "warnings.h" +#include <init.h> +#include <rpc/server.h> +#include <scheduler.h> +#include <ui_interface.h> +#include <util.h> +#include <warnings.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" +#include <wallet/wallet.h> #endif #include <stdint.h> @@ -261,7 +261,7 @@ private: void startThread(); }; -#include "bitcoin.moc" +#include <qt/bitcoin.moc> BitcoinCore::BitcoinCore(): QObject() diff --git a/src/qt/bitcoinaddressvalidator.cpp b/src/qt/bitcoinaddressvalidator.cpp index 362a71f04d..e8d174bf29 100644 --- a/src/qt/bitcoinaddressvalidator.cpp +++ b/src/qt/bitcoinaddressvalidator.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinaddressvalidator.h" +#include <qt/bitcoinaddressvalidator.h> -#include "base58.h" +#include <base58.h> /* Base58 characters are: "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 73eb35a54e..0b21349c54 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinamountfield.h" +#include <qt/bitcoinamountfield.h> -#include "bitcoinunits.h" -#include "guiconstants.h" -#include "qvaluecombobox.h" +#include <qt/bitcoinunits.h> +#include <qt/guiconstants.h> +#include <qt/qvaluecombobox.h> #include <QApplication> #include <QAbstractSpinBox> @@ -188,7 +188,7 @@ Q_SIGNALS: void valueChanged(); }; -#include "bitcoinamountfield.moc" +#include <qt/bitcoinamountfield.moc> BitcoinAmountField::BitcoinAmountField(QWidget *parent) : QWidget(parent), diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index 2f03a3d171..659ecb416b 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINAMOUNTFIELD_H #define BITCOIN_QT_BITCOINAMOUNTFIELD_H -#include "amount.h" +#include <amount.h> #include <QWidget> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index dc55141900..f925ec5359 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -2,39 +2,35 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#include "bitcoingui.h" - -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "modaloverlay.h" -#include "networkstyle.h" -#include "notificator.h" -#include "openuridialog.h" -#include "optionsdialog.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "rpcconsole.h" -#include "utilitydialog.h" +#include <qt/bitcoingui.h> + +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/modaloverlay.h> +#include <qt/networkstyle.h> +#include <qt/notificator.h> +#include <qt/openuridialog.h> +#include <qt/optionsdialog.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/rpcconsole.h> +#include <qt/utilitydialog.h> #ifdef ENABLE_WALLET -#include "walletframe.h" -#include "walletmodel.h" +#include <qt/walletframe.h> +#include <qt/walletmodel.h> #endif // ENABLE_WALLET #ifdef Q_OS_MAC -#include "macdockiconhandler.h" +#include <qt/macdockiconhandler.h> #endif -#include "chainparams.h" -#include "init.h" -#include "ui_interface.h" -#include "util.h" +#include <chainparams.h> +#include <init.h> +#include <ui_interface.h> +#include <util.h> #include <iostream> diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index aa45ea1f0a..1647975fec 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -6,10 +6,10 @@ #define BITCOIN_QT_BITCOINGUI_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "amount.h" +#include <amount.h> #include <QLabel> #include <QMainWindow> diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 4da414a4e3..c8614fdfe0 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinunits.h" +#include <qt/bitcoinunits.h> -#include "primitives/transaction.h" +#include <primitives/transaction.h> #include <QStringList> diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 6ef37de380..3f5a7fd32d 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINUNITS_H #define BITCOIN_QT_BITCOINUNITS_H -#include "amount.h" +#include <amount.h> #include <QAbstractListModel> #include <QString> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 52ce11cefd..78174b90a1 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -2,23 +2,23 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "clientmodel.h" - -#include "bantablemodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "peertablemodel.h" - -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "clientversion.h" -#include "validation.h" -#include "net.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "util.h" -#include "warnings.h" +#include <qt/clientmodel.h> + +#include <qt/bantablemodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/peertablemodel.h> + +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <clientversion.h> +#include <validation.h> +#include <net.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <util.h> +#include <warnings.h> #include <stdint.h> diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 3ca43eae22..49f4e74a9f 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -2,24 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coincontroldialog.h" -#include "ui_coincontroldialog.h" - -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "txmempool.h" -#include "walletmodel.h" - -#include "wallet/coincontrol.h" -#include "init.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "validation.h" // For mempool -#include "wallet/fees.h" -#include "wallet/wallet.h" +#include <qt/coincontroldialog.h> +#include <qt/forms/ui_coincontroldialog.h> + +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <txmempool.h> +#include <qt/walletmodel.h> + +#include <wallet/coincontrol.h> +#include <init.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <validation.h> // For mempool +#include <wallet/fees.h> +#include <wallet/wallet.h> #include <QApplication> #include <QCheckBox> @@ -28,9 +28,7 @@ #include <QFlags> #include <QIcon> #include <QSettings> -#include <QString> #include <QTreeWidget> -#include <QTreeWidgetItem> QList<CAmount> CoinControlDialog::payAmounts; CCoinControl* CoinControlDialog::coinControl = new CCoinControl(); @@ -582,7 +580,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) QString toolTipDust = tr("This label turns red if any recipient receives an amount smaller than the current dust threshold."); // how many satoshis the estimated fee can vary per byte we guess wrong - double dFeeVary = (double)nPayFee / nBytes; + double dFeeVary = (nBytes != 0) ? (double)nPayFee / nBytes : 0; QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 4949c91771..51b0a42907 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_COINCONTROLDIALOG_H #define BITCOIN_QT_COINCONTROLDIALOG_H -#include "amount.h" +#include <amount.h> #include <QAbstractButton> #include <QAction> diff --git a/src/qt/coincontroltreewidget.cpp b/src/qt/coincontroltreewidget.cpp index 88510b6168..b628824793 100644 --- a/src/qt/coincontroltreewidget.cpp +++ b/src/qt/coincontroltreewidget.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coincontroltreewidget.h" -#include "coincontroldialog.h" +#include <qt/coincontroltreewidget.h> +#include <qt/coincontroldialog.h> CoinControlTreeWidget::CoinControlTreeWidget(QWidget *parent) : QTreeWidget(parent) diff --git a/src/qt/csvmodelwriter.cpp b/src/qt/csvmodelwriter.cpp index df59927782..702757147b 100644 --- a/src/qt/csvmodelwriter.cpp +++ b/src/qt/csvmodelwriter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "csvmodelwriter.h" +#include <qt/csvmodelwriter.h> #include <QAbstractItemModel> #include <QFile> diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index bb03c12f2d..0437f81a7e 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "editaddressdialog.h" -#include "ui_editaddressdialog.h" +#include <qt/editaddressdialog.h> +#include <qt/forms/ui_editaddressdialog.h> -#include "addresstablemodel.h" -#include "guiutil.h" +#include <qt/addresstablemodel.h> +#include <qt/guiutil.h> #include <QDataWidgetMapper> #include <QMessageBox> diff --git a/src/qt/forms/askpassphrasedialog.ui b/src/qt/forms/askpassphrasedialog.ui index a2105ecd0a..69803989cd 100644 --- a/src/qt/forms/askpassphrasedialog.ui +++ b/src/qt/forms/askpassphrasedialog.ui @@ -93,6 +93,13 @@ </widget> </item> <item row="3" column="1"> + <widget class="QCheckBox" name="toggleShowPasswordButton"> + <property name="text"> + <string>Show password</string> + </property> + </widget> + </item> + <item row="4" column="1"> <widget class="QLabel" name="capsLabel"> <property name="font"> <font> diff --git a/src/qt/forms/coincontroldialog.ui b/src/qt/forms/coincontroldialog.ui index 1ea00eb5c3..d1237ad283 100644 --- a/src/qt/forms/coincontroldialog.ui +++ b/src/qt/forms/coincontroldialog.ui @@ -487,7 +487,7 @@ <customwidget> <class>CoinControlTreeWidget</class> <extends>QTreeWidget</extends> - <header>coincontroltreewidget.h</header> + <header>qt/coincontroltreewidget.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 093e644bdc..bba822882e 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1424,7 +1424,7 @@ <customwidget> <class>TrafficGraphWidget</class> <extends>QWidget</extends> - <header>trafficgraphwidget.h</header> + <header>qt/trafficgraphwidget.h</header> <container>1</container> <slots> <slot>clear()</slot> diff --git a/src/qt/forms/editaddressdialog.ui b/src/qt/forms/editaddressdialog.ui index c1aea36338..e968271107 100644 --- a/src/qt/forms/editaddressdialog.ui +++ b/src/qt/forms/editaddressdialog.ui @@ -71,7 +71,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index 65a7a6c77e..fdc52dc455 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -367,7 +367,7 @@ QLabel { color: rgb(40,40,40); }</string> <customwidget> <class>ModalOverlay</class> <extends>QWidget</extends> - <header>modaloverlay.h</header> + <header>qt/modaloverlay.h</header> <container>1</container> </customwidget> </customwidgets> diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 7fce858bd2..0e1048bc07 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -77,7 +77,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index e31bfee05e..a3721991ee 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -824,12 +824,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>QValueComboBox</class> <extends>QComboBox</extends> - <header>qvaluecombobox.h</header> + <header>qt/qvaluecombobox.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 03fcb2fb50..58f030ebf0 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -28,23 +28,6 @@ <layout class="QVBoxLayout" name="verticalLayout_3"> <item> <layout class="QGridLayout" name="gridLayout"> - <item row="7" column="2"> - <widget class="QCheckBox" name="reuseAddress"> - <property name="toolTip"> - <string>Reuse one of the previously used receiving addresses. Reusing addresses has security and privacy issues. Do not use this unless re-generating a payment request made before.</string> - </property> - <property name="text"> - <string>R&euse an existing receiving address (not recommended)</string> - </property> - </widget> - </item> - <item row="7" column="0"> - <widget class="QLabel" name="label_4"> - <property name="text"> - <string/> - </property> - </widget> - </item> <item row="6" column="0"> <widget class="QLabel" name="label_3"> <property name="toolTip"> @@ -127,7 +110,7 @@ </property> </widget> </item> - <item row="8" column="2"> + <item row="7" column="2"> <layout class="QHBoxLayout" name="horizontalLayout"> <item> <widget class="QPushButton" name="receiveButton"> @@ -184,7 +167,7 @@ </item> </layout> </item> - <item row="8" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="label_7"> <property name="text"> <string/> @@ -316,7 +299,7 @@ <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> <container>1</container> </customwidget> </customwidgets> @@ -324,7 +307,6 @@ <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> <tabstop>reqMessage</tabstop> - <tabstop>reuseAddress</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> <tabstop>recentRequestsView</tabstop> diff --git a/src/qt/forms/receiverequestdialog.ui b/src/qt/forms/receiverequestdialog.ui index 4163f4189c..dbe966b241 100644 --- a/src/qt/forms/receiverequestdialog.ui +++ b/src/qt/forms/receiverequestdialog.ui @@ -127,7 +127,7 @@ <customwidget> <class>QRImageWidget</class> <extends>QLabel</extends> - <header>receiverequestdialog.h</header> + <header>qt/receiverequestdialog.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index a0e48334c1..195a5560f7 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -1108,10 +1108,10 @@ <item> <widget class="QCheckBox" name="optInRBF"> <property name="text"> - <string>Request Replace-By-Fee</string> + <string>Enable Replace-By-Fee</string> </property> <property name="toolTip"> - <string>Indicates that the sender may wish to replace this transaction with a new one paying higher fees (prior to being confirmed).</string> + <string>With Replace-By-Fee (BIP-125) you can increase a transaction's fee after it is sent. Without this, a higher fee may be recommended to compensate for increased transaction delay risk.</string> </property> </widget> </item> @@ -1264,12 +1264,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> <container>1</container> </customwidget> </customwidgets> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index df06f36833..3c699abc6a 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -163,7 +163,7 @@ </widget> </item> <item row="2" column="1"> - <layout class="QHBoxLayout" name="horizontalLayoutAmount" stretch="0,1"> + <layout class="QHBoxLayout" name="horizontalLayoutAmount" stretch="0,1,0"> <item> <widget class="BitcoinAmountField" name="payAmount"/> </item> @@ -177,6 +177,13 @@ </property> </widget> </item> + <item> + <widget class="QPushButton" name="useAvailableBalanceButton"> + <property name="text"> + <string>Use available balance</string> + </property> + </widget> + </item> </layout> </item> <item row="3" column="0"> @@ -1248,12 +1255,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> </customwidget> </customwidgets> <tabstops> diff --git a/src/qt/forms/signverifymessagedialog.ui b/src/qt/forms/signverifymessagedialog.ui index 92f6430c51..c9ddd757c1 100644 --- a/src/qt/forms/signverifymessagedialog.ui +++ b/src/qt/forms/signverifymessagedialog.ui @@ -380,7 +380,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index d520d7d4be..eb5d3304af 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -2,21 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "guiutil.h" - -#include "bitcoinaddressvalidator.h" -#include "bitcoinunits.h" -#include "qvalidatedlineedit.h" -#include "walletmodel.h" - -#include "fs.h" -#include "primitives/transaction.h" -#include "init.h" -#include "policy/policy.h" -#include "protocol.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" +#include <qt/guiutil.h> + +#include <qt/bitcoinaddressvalidator.h> +#include <qt/bitcoinunits.h> +#include <qt/qvalidatedlineedit.h> +#include <qt/walletmodel.h> + +#include <primitives/transaction.h> +#include <init.h> +#include <policy/policy.h> +#include <protocol.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> #ifdef WIN32 #ifdef _WIN32_WINNT @@ -31,9 +30,9 @@ #ifndef NOMINMAX #define NOMINMAX #endif -#include "shellapi.h" -#include "shlobj.h" -#include "shlwapi.h" +#include <shellapi.h> +#include <shlobj.h> +#include <shlwapi.h> #endif #include <boost/scoped_array.hpp> @@ -984,6 +983,18 @@ QString formatNiceTimeOffset(qint64 secs) return timeBehindText; } +QString formatBytes(uint64_t bytes) +{ + if(bytes < 1024) + return QString(QObject::tr("%1 B")).arg(bytes); + if(bytes < 1024 * 1024) + return QString(QObject::tr("%1 KB")).arg(bytes / 1024); + if(bytes < 1024 * 1024 * 1024) + return QString(QObject::tr("%1 MB")).arg(bytes / 1024 / 1024); + + return QString(QObject::tr("%1 GB")).arg(bytes / 1024 / 1024 / 1024); +} + void ClickableLabel::mouseReleaseEvent(QMouseEvent *event) { Q_EMIT clicked(event->pos()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index d10818d0c8..d0ab491952 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_QT_GUIUTIL_H #define BITCOIN_QT_GUIUTIL_H -#include "amount.h" -#include "fs.h" +#include <amount.h> +#include <fs.h> #include <QEvent> #include <QHeaderView> @@ -199,6 +199,8 @@ namespace GUIUtil QString formatNiceTimeOffset(qint64 secs); + QString formatBytes(uint64_t bytes); + class ClickableLabel : public QLabel { Q_OBJECT diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 0ff95d8502..7f8a8394e6 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -3,16 +3,16 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "fs.h" -#include "intro.h" -#include "ui_intro.h" +#include <fs.h> +#include <qt/intro.h> +#include <qt/forms/ui_intro.h> -#include "guiutil.h" +#include <qt/guiutil.h> -#include "util.h" +#include <util.h> #include <QFileDialog> #include <QSettings> @@ -60,7 +60,7 @@ private: Intro *intro; }; -#include "intro.moc" +#include <qt/intro.moc> FreespaceChecker::FreespaceChecker(Intro *_intro) { @@ -214,7 +214,10 @@ bool Intro::pickDataDirectory() } dataDir = intro.getDataDirectory(); try { - TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir)); + if (TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir))) { + // If a new data directory has been created, make wallets subdirectory too + TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir) / "wallets"); + } break; } catch (const fs::filesystem_error&) { QMessageBox::critical(0, tr(PACKAGE_NAME), diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index e32a0bdda8..b707b0dda9 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "modaloverlay.h" -#include "ui_modaloverlay.h" +#include <qt/modaloverlay.h> +#include <qt/forms/ui_modaloverlay.h> -#include "guiutil.h" +#include <qt/guiutil.h> -#include "chainparams.h" +#include <chainparams.h> #include <QResizeEvent> #include <QPropertyAnimation> diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index 4b81c54d36..42ae3ae566 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "networkstyle.h" +#include <qt/networkstyle.h> -#include "guiconstants.h" +#include <qt/guiconstants.h> #include <QApplication> diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 937928315b..f4383bf679 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -2,11 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "notificator.h" +#include <qt/notificator.h> #include <QApplication> #include <QByteArray> -#include <QIcon> #include <QImageWriter> #include <QMessageBox> #include <QMetaType> @@ -24,7 +23,7 @@ // #define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 #ifdef Q_OS_MAC #include <ApplicationServices/ApplicationServices.h> -#include "macnotificationhandler.h" +#include <qt/macnotificationhandler.h> #endif diff --git a/src/qt/notificator.h b/src/qt/notificator.h index 67f2b1df69..ef6f5c7ec5 100644 --- a/src/qt/notificator.h +++ b/src/qt/notificator.h @@ -6,7 +6,7 @@ #define BITCOIN_QT_NOTIFICATOR_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <QIcon> diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 3ee656d470..40671922f4 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "openuridialog.h" -#include "ui_openuridialog.h" +#include <qt/openuridialog.h> +#include <qt/forms/ui_openuridialog.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> #include <QUrl> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 53e2e5053c..a0f78d5ead 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -3,19 +3,19 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "optionsdialog.h" -#include "ui_optionsdialog.h" +#include <qt/optionsdialog.h> +#include <qt/forms/ui_optionsdialog.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "validation.h" // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS -#include "netbase.h" -#include "txdb.h" // for -dbcache defaults +#include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS +#include <netbase.h> +#include <txdb.h> // for -dbcache defaults #include <QDataWidgetMapper> #include <QDir> @@ -338,7 +338,7 @@ QValidator::State ProxyAddressValidator::validate(QString &input, int &pos) cons { Q_UNUSED(pos); // Validate the proxy - CService serv(LookupNumeric(input.toStdString().c_str(), 9050)); + CService serv(LookupNumeric(input.toStdString().c_str(), DEFAULT_GUI_PROXY_PORT)); proxyType addrProxy = proxyType(serv, true); if (addrProxy.IsValid()) return QValidator::Acceptable; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index fb8c60d100..c9e4f517c7 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -3,31 +3,32 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "optionsmodel.h" +#include <qt/optionsmodel.h> -#include "bitcoinunits.h" -#include "guiutil.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> -#include "amount.h" -#include "init.h" -#include "validation.h" // For DEFAULT_SCRIPTCHECK_THREADS -#include "net.h" -#include "netbase.h" -#include "txdb.h" // for -dbcache defaults -#include "intro.h" +#include <init.h> +#include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS +#include <net.h> +#include <netbase.h> +#include <txdb.h> // for -dbcache defaults +#include <qt/intro.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <wallet/wallet.h> +#include <wallet/walletdb.h> #endif #include <QNetworkProxy> #include <QSettings> #include <QStringList> +const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; + OptionsModel::OptionsModel(QObject *parent, bool resetSettings) : QAbstractListModel(parent) { @@ -124,8 +125,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fUseProxy")) settings.setValue("fUseProxy", false); - if (!settings.contains("addrProxy") || !settings.value("addrProxy").toString().contains(':')) - settings.setValue("addrProxy", "127.0.0.1:9050"); + if (!settings.contains("addrProxy")) + settings.setValue("addrProxy", QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST, DEFAULT_GUI_PROXY_PORT)); // Only try to set -proxy, if user has enabled fUseProxy if (settings.value("fUseProxy").toBool() && !gArgs.SoftSetArg("-proxy", settings.value("addrProxy").toString().toStdString())) addOverriddenOption("-proxy"); @@ -134,8 +135,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fUseSeparateProxyTor")) settings.setValue("fUseSeparateProxyTor", false); - if (!settings.contains("addrSeparateProxyTor") || !settings.value("addrSeparateProxyTor").toString().contains(':')) - settings.setValue("addrSeparateProxyTor", "127.0.0.1:9050"); + if (!settings.contains("addrSeparateProxyTor")) + settings.setValue("addrSeparateProxyTor", QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST, DEFAULT_GUI_PROXY_PORT)); // Only try to set -onion, if user has enabled fUseSeparateProxyTor if (settings.value("fUseSeparateProxyTor").toBool() && !gArgs.SoftSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString())) addOverriddenOption("-onion"); @@ -200,6 +201,33 @@ int OptionsModel::rowCount(const QModelIndex & parent) const return OptionIDRowCount; } +struct ProxySetting { + bool is_set; + QString ip; + QString port; +}; + +static ProxySetting GetProxySetting(QSettings &settings, const QString &name) +{ + static const ProxySetting default_val = {false, DEFAULT_GUI_PROXY_HOST, QString("%1").arg(DEFAULT_GUI_PROXY_PORT)}; + // Handle the case that the setting is not set at all + if (!settings.contains(name)) { + return default_val; + } + // contains IP at index 0 and port at index 1 + QStringList ip_port = settings.value(name).toString().split(":", QString::SkipEmptyParts); + if (ip_port.size() == 2) { + return {true, ip_port.at(0), ip_port.at(1)}; + } else { // Invalid: return default + return default_val; + } +} + +static void SetProxySetting(QSettings &settings, const QString &name, const ProxySetting &ip_port) +{ + settings.setValue(name, ip_port.ip + ":" + ip_port.port); +} + // read QSettings values and return them QVariant OptionsModel::data(const QModelIndex & index, int role) const { @@ -226,30 +254,18 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const // default proxy case ProxyUse: return settings.value("fUseProxy", false); - case ProxyIP: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(0); - } - case ProxyPort: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(1); - } + case ProxyIP: + return GetProxySetting(settings, "addrProxy").ip; + case ProxyPort: + return GetProxySetting(settings, "addrProxy").port; // separate Tor proxy case ProxyUseTor: return settings.value("fUseSeparateProxyTor", false); - case ProxyIPTor: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(0); - } - case ProxyPortTor: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(1); - } + case ProxyIPTor: + return GetProxySetting(settings, "addrSeparateProxyTor").ip; + case ProxyPortTor: + return GetProxySetting(settings, "addrSeparateProxyTor").port; #ifdef ENABLE_WALLET case SpendZeroConfChange: @@ -314,25 +330,19 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in } break; case ProxyIP: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed IP - if (!settings.contains("addrProxy") || strlIpPort.at(0) != value.toString()) { - // construct new value from new IP and current port - QString strNewValue = value.toString() + ":" + strlIpPort.at(1); - settings.setValue("addrProxy", strNewValue); + auto ip_port = GetProxySetting(settings, "addrProxy"); + if (!ip_port.is_set || ip_port.ip != value.toString()) { + ip_port.ip = value.toString(); + SetProxySetting(settings, "addrProxy", ip_port); setRestartRequired(true); } } break; case ProxyPort: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed port - if (!settings.contains("addrProxy") || strlIpPort.at(1) != value.toString()) { - // construct new value from current IP and new port - QString strNewValue = strlIpPort.at(0) + ":" + value.toString(); - settings.setValue("addrProxy", strNewValue); + auto ip_port = GetProxySetting(settings, "addrProxy"); + if (!ip_port.is_set || ip_port.port != value.toString()) { + ip_port.port = value.toString(); + SetProxySetting(settings, "addrProxy", ip_port); setRestartRequired(true); } } @@ -346,25 +356,19 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in } break; case ProxyIPTor: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed IP - if (!settings.contains("addrSeparateProxyTor") || strlIpPort.at(0) != value.toString()) { - // construct new value from new IP and current port - QString strNewValue = value.toString() + ":" + strlIpPort.at(1); - settings.setValue("addrSeparateProxyTor", strNewValue); + auto ip_port = GetProxySetting(settings, "addrSeparateProxyTor"); + if (!ip_port.is_set || ip_port.ip != value.toString()) { + ip_port.ip = value.toString(); + SetProxySetting(settings, "addrSeparateProxyTor", ip_port); setRestartRequired(true); } } break; case ProxyPortTor: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed port - if (!settings.contains("addrSeparateProxyTor") || strlIpPort.at(1) != value.toString()) { - // construct new value from current IP and new port - QString strNewValue = strlIpPort.at(0) + ":" + value.toString(); - settings.setValue("addrSeparateProxyTor", strNewValue); + auto ip_port = GetProxySetting(settings, "addrSeparateProxyTor"); + if (!ip_port.is_set || ip_port.port != value.toString()) { + ip_port.port = value.toString(); + SetProxySetting(settings, "addrSeparateProxyTor", ip_port); setRestartRequired(true); } } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 0ac82a4148..f8782dd204 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_OPTIONSMODEL_H #define BITCOIN_QT_OPTIONSMODEL_H -#include "amount.h" +#include <amount.h> #include <QAbstractListModel> @@ -13,6 +13,9 @@ QT_BEGIN_NAMESPACE class QNetworkProxy; QT_END_NAMESPACE +extern const char *DEFAULT_GUI_PROXY_HOST; +static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; + /** Interface from Qt to configuration data structure for Bitcoin client. To Qt, the options are presented as a list with the different options laid out vertically. diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index ba1839e7b4..7818607d30 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "overviewpage.h" -#include "ui_overviewpage.h" - -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "transactionfilterproxy.h" -#include "transactiontablemodel.h" -#include "walletmodel.h" +#include <qt/overviewpage.h> +#include <qt/forms/ui_overviewpage.h> + +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/transactionfilterproxy.h> +#include <qt/transactiontablemodel.h> +#include <qt/walletmodel.h> #include <QAbstractItemDelegate> #include <QPainter> @@ -106,7 +106,7 @@ public: const PlatformStyle *platformStyle; }; -#include "overviewpage.moc" +#include <qt/overviewpage.moc> OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index d76b651ce6..1e09d907b0 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_OVERVIEWPAGE_H #define BITCOIN_QT_OVERVIEWPAGE_H -#include "amount.h" +#include <amount.h> #include <QWidget> #include <memory> diff --git a/src/qt/paymentrequestplus.cpp b/src/qt/paymentrequestplus.cpp index d3799f59ab..4503728882 100644 --- a/src/qt/paymentrequestplus.cpp +++ b/src/qt/paymentrequestplus.cpp @@ -7,9 +7,9 @@ // with some extra methods // -#include "paymentrequestplus.h" +#include <qt/paymentrequestplus.h> -#include "util.h" +#include <util.h> #include <stdexcept> @@ -194,8 +194,7 @@ bool PaymentRequestPlus::getMerchant(X509_STORE* certStore, QString& merchant) c qWarning() << "PaymentRequestPlus::getMerchant: SSL error: " << err.what(); } - if (website) - delete[] website; + delete[] website; X509_STORE_CTX_free(store_ctx); for (unsigned int i = 0; i < certs.size(); i++) X509_free(certs[i]); diff --git a/src/qt/paymentrequestplus.h b/src/qt/paymentrequestplus.h index a2fea3fdc6..e1388f2b5d 100644 --- a/src/qt/paymentrequestplus.h +++ b/src/qt/paymentrequestplus.h @@ -7,10 +7,10 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "paymentrequest.pb.h" +#include <qt/paymentrequest.pb.h> #pragma GCC diagnostic pop -#include "base58.h" +#include <base58.h> #include <openssl/x509.h> diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 169684cf6d..510a3783ae 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentserver.h" +#include <qt/paymentserver.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "base58.h" -#include "chainparams.h" -#include "policy/policy.h" -#include "ui_interface.h" -#include "util.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <chainparams.h> +#include <policy/policy.h> +#include <ui_interface.h> +#include <util.h> +#include <wallet/wallet.h> #include <cstdlib> @@ -362,8 +362,7 @@ void PaymentServer::initNetManager() { if (!optionsModel) return; - if (netManager != nullptr) - delete netManager; + delete netManager; // netManager is used to fetch paymentrequests given in bitcoin: URIs netManager = new QNetworkAccessManager(this); diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 98b2364b92..9adef9743d 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -32,8 +32,8 @@ // sends them to the server. // -#include "paymentrequestplus.h" -#include "walletmodel.h" +#include <qt/paymentrequestplus.h> +#include <qt/walletmodel.h> #include <QObject> #include <QString> diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 42934f8055..cfafab2c8a 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "peertablemodel.h" +#include <qt/peertablemodel.h> -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> -#include "validation.h" // for cs_main -#include "sync.h" +#include <validation.h> // for cs_main +#include <sync.h> #include <QDebug> #include <QList> @@ -33,6 +33,10 @@ bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombine return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; case PeerTableModel::Ping: return pLeft->dMinPing < pRight->dMinPing; + case PeerTableModel::Sent: + return pLeft->nSendBytes < pRight->nSendBytes; + case PeerTableModel::Received: + return pLeft->nRecvBytes < pRight->nRecvBytes; } return false; @@ -114,7 +118,7 @@ PeerTableModel::PeerTableModel(ClientModel *parent) : clientModel(parent), timer(0) { - columns << tr("NodeId") << tr("Node/Service") << tr("User Agent") << tr("Ping"); + columns << tr("NodeId") << tr("Node/Service") << tr("Ping") << tr("Sent") << tr("Received") << tr("User Agent"); priv.reset(new PeerTablePriv()); // default to unsorted priv->sortColumn = -1; @@ -173,10 +177,20 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const return QString::fromStdString(rec->nodeStats.cleanSubVer); case Ping: return GUIUtil::formatPingTime(rec->nodeStats.dMinPing); + case Sent: + return GUIUtil::formatBytes(rec->nodeStats.nSendBytes); + case Received: + return GUIUtil::formatBytes(rec->nodeStats.nRecvBytes); } } else if (role == Qt::TextAlignmentRole) { - if (index.column() == Ping) - return (QVariant)(Qt::AlignRight | Qt::AlignVCenter); + switch (index.column()) { + case Ping: + case Sent: + case Received: + return QVariant(Qt::AlignRight | Qt::AlignVCenter); + default: + return QVariant(); + } } return QVariant(); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index cc47b67ec9..e41fe4bb03 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_QT_PEERTABLEMODEL_H #define BITCOIN_QT_PEERTABLEMODEL_H -#include "net_processing.h" // For CNodeStateStats -#include "net.h" +#include <net_processing.h> // For CNodeStateStats +#include <net.h> #include <QAbstractTableModel> #include <QStringList> @@ -55,8 +55,10 @@ public: enum ColumnIndex { NetNodeId = 0, Address = 1, - Subversion = 2, - Ping = 3 + Ping = 2, + Sent = 3, + Received = 4, + Subversion = 5 }; /** @name Methods overridden from QAbstractTableModel diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index 1f4e1a442f..0e1bd59ada 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -2,16 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "platformstyle.h" +#include <qt/platformstyle.h> -#include "guiconstants.h" +#include <qt/guiconstants.h> #include <QApplication> #include <QColor> -#include <QIcon> #include <QImage> #include <QPalette> -#include <QPixmap> static const struct { const char *platformId; diff --git a/src/qt/qvalidatedlineedit.cpp b/src/qt/qvalidatedlineedit.cpp index 179ecdc8b3..4d6d5891c9 100644 --- a/src/qt/qvalidatedlineedit.cpp +++ b/src/qt/qvalidatedlineedit.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "qvalidatedlineedit.h" +#include <qt/qvalidatedlineedit.h> -#include "bitcoinaddressvalidator.h" -#include "guiconstants.h" +#include <qt/bitcoinaddressvalidator.h> +#include <qt/guiconstants.h> QValidatedLineEdit::QValidatedLineEdit(QWidget *parent) : QLineEdit(parent), diff --git a/src/qt/qvaluecombobox.cpp b/src/qt/qvaluecombobox.cpp index a3b9b994ec..af5f6f8fd0 100644 --- a/src/qt/qvaluecombobox.cpp +++ b/src/qt/qvaluecombobox.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "qvaluecombobox.h" +#include <qt/qvaluecombobox.h> QValueComboBox::QValueComboBox(QWidget *parent) : QComboBox(parent), role(Qt::UserRole) diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 84f43266e1..e9dcbc0a72 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -2,22 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "receivecoinsdialog.h" -#include "ui_receivecoinsdialog.h" - -#include "addressbookpage.h" -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "receiverequestdialog.h" -#include "recentrequeststablemodel.h" -#include "walletmodel.h" +#include <qt/receivecoinsdialog.h> +#include <qt/forms/ui_receivecoinsdialog.h> + +#include <qt/addressbookpage.h> +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/receiverequestdialog.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/walletmodel.h> #include <QAction> #include <QCursor> -#include <QItemSelection> #include <QMessageBox> #include <QScrollBar> #include <QTextDocument> @@ -106,7 +104,6 @@ void ReceiveCoinsDialog::clear() ui->reqAmount->clear(); ui->reqLabel->setText(""); ui->reqMessage->setText(""); - ui->reuseAddress->setChecked(false); updateDisplayUnit(); } @@ -135,25 +132,8 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString address; QString label = ui->reqLabel->text(); - if(ui->reuseAddress->isChecked()) - { - /* Choose existing receiving address */ - AddressBookPage dlg(platformStyle, AddressBookPage::ForSelection, AddressBookPage::ReceivingTab, this); - dlg.setModel(model->getAddressTableModel()); - if(dlg.exec()) - { - address = dlg.getReturnValue(); - if(label.isEmpty()) /* If no label provided, use the previously used label */ - { - label = model->getAddressTableModel()->labelForAddress(address); - } - } else { - return; - } - } else { - /* Generate new receiving address */ - address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); - } + /* Generate new receiving address */ + address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); SendCoinsRecipient info(address, label, ui->reqAmount->value(), ui->reqMessage->text()); ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 385f98565c..3687cca87b 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_RECEIVECOINSDIALOG_H #define BITCOIN_QT_RECEIVECOINSDIALOG_H -#include "guiutil.h" +#include <qt/guiutil.h> #include <QDialog> #include <QHeaderView> diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 4aa6375d8a..d30e9b2dca 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -2,14 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "receiverequestdialog.h" -#include "ui_receiverequestdialog.h" +#include <qt/receiverequestdialog.h> +#include <qt/forms/ui_receiverequestdialog.h> -#include "bitcoinunits.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "walletmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> #include <QClipboard> #include <QDrag> @@ -22,7 +21,7 @@ #endif #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" /* for USE_QRCODE */ +#include <config/bitcoin-config.h> /* for USE_QRCODE */ #endif #ifdef USE_QRCODE diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 1a9b165237..23b81135e2 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_RECEIVEREQUESTDIALOG_H #define BITCOIN_QT_RECEIVEREQUESTDIALOG_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QDialog> #include <QImage> diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 1c4f7aca86..4d6e401d0d 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "recentrequeststablemodel.h" +#include <qt/recentrequeststablemodel.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "clientversion.h" -#include "streams.h" +#include <clientversion.h> +#include <streams.h> RecentRequestsTableModel::RecentRequestsTableModel(CWallet *wallet, WalletModel *parent) : diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 0c02968f92..c88b8c4426 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H #define BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index d895fc1663..0b90205270 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -3,21 +3,20 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "rpcconsole.h" -#include "ui_debugwindow.h" +#include <qt/rpcconsole.h> +#include <qt/forms/ui_debugwindow.h> -#include "bantablemodel.h" -#include "clientmodel.h" -#include "guiutil.h" -#include "platformstyle.h" -#include "chainparams.h" -#include "netbase.h" -#include "rpc/server.h" -#include "rpc/client.h" -#include "util.h" +#include <qt/bantablemodel.h> +#include <qt/clientmodel.h> +#include <qt/platformstyle.h> +#include <chainparams.h> +#include <netbase.h> +#include <rpc/server.h> +#include <rpc/client.h> +#include <util.h> #include <openssl/crypto.h> @@ -35,7 +34,6 @@ #include <QScrollBar> #include <QSettings> #include <QSignalMapper> -#include <QThread> #include <QTime> #include <QTimer> #include <QStringList> @@ -125,7 +123,7 @@ public: }; -#include "rpcconsole.moc" +#include <qt/rpcconsole.moc> /** * Split shell command line into a list of arguments and optionally execute the command(s). @@ -392,11 +390,37 @@ void RPCExecutor::request(const QString &command) { std::string result; std::string executableCommand = command.toStdString() + "\n"; + + // Catch the console-only-help command before RPC call is executed and reply with help text as-if a RPC reply. + if(executableCommand == "help-console\n") + { + Q_EMIT reply(RPCConsole::CMD_REPLY, QString(("\n" + "This console accepts RPC commands using the standard syntax.\n" + " example: getblockhash 0\n\n" + + "This console can also accept RPC commands using parenthesized syntax.\n" + " example: getblockhash(0)\n\n" + + "Commands may be nested when specified with the parenthesized syntax.\n" + " example: getblock(getblockhash(0) 1)\n\n" + + "A space or a comma can be used to delimit arguments for either syntax.\n" + " example: getblockhash 0\n" + " getblockhash,0\n\n" + + "Named results can be queried with a non-quoted key string in brackets.\n" + " example: getblock(getblockhash(0) true)[tx]\n\n" + + "Results without keys can be queried using an integer in brackets.\n" + " example: getblock(getblockhash(0),true)[tx][0]\n\n"))); + return; + } if(!RPCConsole::RPCExecuteCommandLine(result, executableCommand)) { Q_EMIT reply(RPCConsole::CMD_ERROR, QString("Parse error: unbalanced ' or \"")); return; } + Q_EMIT reply(RPCConsole::CMD_REPLY, QString::fromStdString(result)); } catch (UniValue& objError) @@ -645,6 +669,7 @@ void RPCConsole::setClientModel(ClientModel *model) wordList << ("help " + commandList[i]).c_str(); } + wordList << "help-console"; wordList.sort(); autoCompleter = new QCompleter(wordList, this); autoCompleter->setModelSorting(QCompleter::CaseSensitivelySortedModel); @@ -750,10 +775,11 @@ void RPCConsole::clear(bool clearHistory) message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + - tr("Type <b>help</b> for an overview of available commands.")) + - "<br><span class=\"secwarning\">" + + tr("Type %1 for an overview of available commands.").arg("<b>help</b>") + "<br>" + + tr("For more information on using this console type %1.").arg("<b>help-console</b>") + + "<br><span class=\"secwarning\"><br>" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramifications of a command.") + - "</span>", + "</span>"), true); } @@ -935,18 +961,6 @@ void RPCConsole::on_sldGraphRange_valueChanged(int value) setTrafficGraphRange(mins); } -QString RPCConsole::FormatBytes(quint64 bytes) -{ - if(bytes < 1024) - return QString(tr("%1 B")).arg(bytes); - if(bytes < 1024 * 1024) - return QString(tr("%1 KB")).arg(bytes / 1024); - if(bytes < 1024 * 1024 * 1024) - return QString(tr("%1 MB")).arg(bytes / 1024 / 1024); - - return QString(tr("%1 GB")).arg(bytes / 1024 / 1024 / 1024); -} - void RPCConsole::setTrafficGraphRange(int mins) { ui->trafficGraph->setGraphRangeMins(mins); @@ -955,8 +969,8 @@ void RPCConsole::setTrafficGraphRange(int mins) void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) { - ui->lblBytesIn->setText(FormatBytes(totalBytesIn)); - ui->lblBytesOut->setText(FormatBytes(totalBytesOut)); + ui->lblBytesIn->setText(GUIUtil::formatBytes(totalBytesIn)); + ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelection &deselected) @@ -1050,8 +1064,8 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); ui->peerLastSend->setText(stats->nodeStats.nLastSend ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastSend) : tr("never")); ui->peerLastRecv->setText(stats->nodeStats.nLastRecv ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastRecv) : tr("never")); - ui->peerBytesSent->setText(FormatBytes(stats->nodeStats.nSendBytes)); - ui->peerBytesRecv->setText(FormatBytes(stats->nodeStats.nRecvBytes)); + ui->peerBytesSent->setText(GUIUtil::formatBytes(stats->nodeStats.nSendBytes)); + ui->peerBytesRecv->setText(GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); ui->peerConnTime->setText(GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nTimeConnected)); ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.dPingTime)); ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStats.dPingWait)); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index da06818f87..5960410cdd 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H -#include "guiutil.h" -#include "peertablemodel.h" +#include <qt/guiutil.h> +#include <qt/peertablemodel.h> -#include "net.h" +#include <net.h> #include <QWidget> #include <QCompleter> @@ -123,7 +123,6 @@ Q_SIGNALS: void cmdRequest(const QString &command); private: - static QString FormatBytes(quint64 bytes); void startExecutor(); void setTrafficGraphRange(int mins); /** show detailed information on ui about selected node */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 6309070fef..9473198dfc 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -2,34 +2,31 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sendcoinsdialog.h" -#include "ui_sendcoinsdialog.h" - -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "coincontroldialog.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "sendcoinsentry.h" -#include "walletmodel.h" - -#include "base58.h" -#include "chainparams.h" -#include "wallet/coincontrol.h" -#include "validation.h" // mempool and minRelayTxFee -#include "ui_interface.h" -#include "txmempool.h" -#include "policy/fees.h" -#include "wallet/fees.h" +#include <qt/sendcoinsdialog.h> +#include <qt/forms/ui_sendcoinsdialog.h> + +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/coincontroldialog.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/sendcoinsentry.h> + +#include <base58.h> +#include <chainparams.h> +#include <wallet/coincontrol.h> +#include <validation.h> // mempool and minRelayTxFee +#include <ui_interface.h> +#include <txmempool.h> +#include <policy/fees.h> +#include <wallet/fees.h> #include <QFontMetrics> -#include <QMessageBox> #include <QScrollBar> #include <QSettings> #include <QTextDocument> -#include <QTimer> static const std::array<int, 9> confTargets = { {2, 4, 6, 12, 24, 48, 144, 504, 1008} }; int getConfTargetForIndex(int index) { @@ -165,7 +162,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) coinControlUpdateLabels(); // fee section - for (const int &n : confTargets) { + for (const int n : confTargets) { ui->confTargetSelector->addItem(tr("%1 (%2 blocks)").arg(GUIUtil::formatNiceTimeOffset(n*Params().GetConsensus().nPowTargetSpacing)).arg(n)); } connect(ui->confTargetSelector, SIGNAL(currentIndexChanged(int)), this, SLOT(updateSmartFeeLabel())); @@ -184,7 +181,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); // set default rbf checkbox state - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + ui->optInRBF->setCheckState(Qt::Checked); // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; @@ -342,12 +339,14 @@ void SendCoinsDialog::on_sendButton_clicked() questionString.append(QString("<span style='font-size:10pt;font-weight:normal;'><br />(=%2)</span>") .arg(alternativeUnits.join(" " + tr("or") + "<br />"))); - if (ui->optInRBF->isChecked()) - { - questionString.append("<hr /><span>"); - questionString.append(tr("This transaction signals replaceability (optin-RBF).")); - questionString.append("</span>"); + questionString.append("<hr /><span>"); + if (ui->optInRBF->isChecked()) { + questionString.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); + } else { + questionString.append(tr("Not signalling Replace-By-Fee, BIP-125.")); } + questionString.append("</span>"); + SendConfirmationDialog confirmationDialog(tr("Confirm send coins"), questionString.arg(formatted.join("<br />")), SEND_CONFIRM_DELAY, this); @@ -402,6 +401,7 @@ SendCoinsEntry *SendCoinsDialog::addEntry() entry->setModel(model); ui->entries->addWidget(entry); connect(entry, SIGNAL(removeEntry(SendCoinsEntry*)), this, SLOT(removeEntry(SendCoinsEntry*))); + connect(entry, SIGNAL(useAvailableBalance(SendCoinsEntry*)), this, SLOT(useAvailableBalance(SendCoinsEntry*))); connect(entry, SIGNAL(payAmountChanged()), this, SLOT(coinControlUpdateLabels())); connect(entry, SIGNAL(subtractFeeFromAmountChanged()), this, SLOT(coinControlUpdateLabels())); @@ -599,6 +599,31 @@ void SendCoinsDialog::on_buttonMinimizeFee_clicked() minimizeFeeSection(true); } +void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry) +{ + // Get CCoinControl instance if CoinControl is enabled or create a new one. + CCoinControl coin_control; + if (model->getOptionsModel()->getCoinControlFeatures()) { + coin_control = *CoinControlDialog::coinControl; + } + + // Calculate available amount to send. + CAmount amount = model->getBalance(&coin_control); + for (int i = 0; i < ui->entries->count(); ++i) { + SendCoinsEntry* e = qobject_cast<SendCoinsEntry*>(ui->entries->itemAt(i)->widget()); + if (e && !e->isHidden() && e != entry) { + amount -= e->getValue().amount; + } + } + + if (amount > 0) { + entry->checkSubtractFeeFromAmount(); + entry->setAmount(amount); + } else { + entry->setAmount(0); + } +} + void SendCoinsDialog::setMinimumFee() { ui->customFee->setValue(GetRequiredFee(1000)); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 70b4aa5a03..300116fd9f 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_SENDCOINSDIALOG_H #define BITCOIN_QT_SENDCOINSDIALOG_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QDialog> #include <QMessageBox> @@ -76,6 +76,7 @@ private Q_SLOTS: void on_buttonChooseFee_clicked(); void on_buttonMinimizeFee_clicked(); void removeEntry(SendCoinsEntry* entry); + void useAvailableBalance(SendCoinsEntry* entry); void updateDisplayUnit(); void coinControlFeatureChanged(bool); void coinControlButtonClicked(); diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index bb0f47b21c..4cf4bb9ef7 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -2,15 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sendcoinsentry.h" -#include "ui_sendcoinsentry.h" +#include <qt/sendcoinsentry.h> +#include <qt/forms/ui_sendcoinsentry.h> -#include "addressbookpage.h" -#include "addresstablemodel.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "walletmodel.h" +#include <qt/addressbookpage.h> +#include <qt/addresstablemodel.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> #include <QApplication> #include <QClipboard> @@ -48,6 +47,7 @@ SendCoinsEntry::SendCoinsEntry(const PlatformStyle *_platformStyle, QWidget *par connect(ui->deleteButton, SIGNAL(clicked()), this, SLOT(deleteClicked())); connect(ui->deleteButton_is, SIGNAL(clicked()), this, SLOT(deleteClicked())); connect(ui->deleteButton_s, SIGNAL(clicked()), this, SLOT(deleteClicked())); + connect(ui->useAvailableBalanceButton, SIGNAL(clicked()), this, SLOT(useAvailableBalanceClicked())); } SendCoinsEntry::~SendCoinsEntry() @@ -112,11 +112,21 @@ void SendCoinsEntry::clear() updateDisplayUnit(); } +void SendCoinsEntry::checkSubtractFeeFromAmount() +{ + ui->checkboxSubtractFeeFromAmount->setChecked(true); +} + void SendCoinsEntry::deleteClicked() { Q_EMIT removeEntry(this); } +void SendCoinsEntry::useAvailableBalanceClicked() +{ + Q_EMIT useAvailableBalance(this); +} + bool SendCoinsEntry::validate() { if (!model) @@ -228,6 +238,11 @@ void SendCoinsEntry::setAddress(const QString &address) ui->payAmount->setFocus(); } +void SendCoinsEntry::setAmount(const CAmount &amount) +{ + ui->payAmount->setValue(amount); +} + bool SendCoinsEntry::isClear() { return ui->payTo->text().isEmpty() && ui->payTo_is->text().isEmpty() && ui->payTo_s->text().isEmpty(); diff --git a/src/qt/sendcoinsentry.h b/src/qt/sendcoinsentry.h index a8be670c2a..b470a0d72e 100644 --- a/src/qt/sendcoinsentry.h +++ b/src/qt/sendcoinsentry.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_SENDCOINSENTRY_H #define BITCOIN_QT_SENDCOINSENTRY_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QStackedWidget> @@ -38,6 +38,7 @@ public: void setValue(const SendCoinsRecipient &value); void setAddress(const QString &address); + void setAmount(const CAmount &amount); /** Set up the tab chain manually, as Qt messes up the tab chain by default in some cases * (issue https://bugreports.qt-project.org/browse/QTBUG-10907). @@ -48,14 +49,17 @@ public: public Q_SLOTS: void clear(); + void checkSubtractFeeFromAmount(); Q_SIGNALS: void removeEntry(SendCoinsEntry *entry); + void useAvailableBalance(SendCoinsEntry* entry); void payAmountChanged(); void subtractFeeFromAmountChanged(); private Q_SLOTS: void deleteClicked(); + void useAvailableBalanceClicked(); void on_payTo_textChanged(const QString &address); void on_addressBookButton_clicked(); void on_pasteButton_clicked(); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index cba9d4da38..4cd0c76b16 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "signverifymessagedialog.h" -#include "ui_signverifymessagedialog.h" - -#include "addressbookpage.h" -#include "guiutil.h" -#include "platformstyle.h" -#include "walletmodel.h" - -#include "base58.h" -#include "init.h" -#include "validation.h" // For strMessageMagic -#include "wallet/wallet.h" +#include <qt/signverifymessagedialog.h> +#include <qt/forms/ui_signverifymessagedialog.h> + +#include <qt/addressbookpage.h> +#include <qt/guiutil.h> +#include <qt/platformstyle.h> +#include <qt/walletmodel.h> + +#include <base58.h> +#include <init.h> +#include <validation.h> // For strMessageMagic +#include <wallet/wallet.h> #include <string> #include <vector> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index a1fbba963c..5aa1de5553 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -3,21 +3,21 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "splashscreen.h" +#include <qt/splashscreen.h> -#include "networkstyle.h" +#include <qt/networkstyle.h> -#include "clientversion.h" -#include "init.h" -#include "util.h" -#include "ui_interface.h" -#include "version.h" +#include <clientversion.h> +#include <init.h> +#include <util.h> +#include <ui_interface.h> +#include <version.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" +#include <wallet/wallet.h> #endif #include <QApplication> diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp index 2a7284b5b2..7a4388e940 100644 --- a/src/qt/test/compattests.cpp +++ b/src/qt/test/compattests.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentrequestplus.h" // this includes protobuf's port.h which defines its own bswap macos +#include <qt/paymentrequestplus.h> // this includes protobuf's port.h which defines its own bswap macos -#include "compattests.h" +#include <qt/test/compattests.h> -#include "compat/byteswap.h" +#include <compat/byteswap.h> void CompatTests::bswapTests() { diff --git a/src/qt/test/paymentservertests.cpp b/src/qt/test/paymentservertests.cpp index 273bd10487..1864604372 100644 --- a/src/qt/test/paymentservertests.cpp +++ b/src/qt/test/paymentservertests.cpp @@ -2,17 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentservertests.h" +#include <qt/test/paymentservertests.h> -#include "optionsmodel.h" -#include "paymentrequestdata.h" +#include <qt/optionsmodel.h> +#include <qt/test/paymentrequestdata.h> -#include "amount.h" -#include "random.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" -#include "utilstrencodings.h" +#include <amount.h> +#include <random.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> +#include <utilstrencodings.h> #include <openssl/x509.h> #include <openssl/x509_vfy.h> diff --git a/src/qt/test/paymentservertests.h b/src/qt/test/paymentservertests.h index faf167f2c6..6d84c38287 100644 --- a/src/qt/test/paymentservertests.h +++ b/src/qt/test/paymentservertests.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H #define BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H -#include "../paymentserver.h" +#include <qt/paymentserver.h> #include <QObject> #include <QTest> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 70fdd4bf58..0596ccf90e 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpcnestedtests.h" - -#include "chainparams.h" -#include "consensus/validation.h" -#include "fs.h" -#include "validation.h" -#include "rpc/register.h" -#include "rpc/server.h" -#include "rpcconsole.h" -#include "test/test_bitcoin.h" -#include "univalue.h" -#include "util.h" +#include <qt/test/rpcnestedtests.h> + +#include <chainparams.h> +#include <consensus/validation.h> +#include <fs.h> +#include <validation.h> +#include <rpc/register.h> +#include <rpc/server.h> +#include <qt/rpcconsole.h> +#include <test/test_bitcoin.h> +#include <univalue.h> +#include <util.h> #include <QDir> #include <QtGlobal> diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 9ad409019f..04a9d124aa 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -8,8 +8,8 @@ #include <QObject> #include <QTest> -#include "txdb.h" -#include "txmempool.h" +#include <txdb.h> +#include <txmempool.h> class RPCNestedTests : public QObject { @@ -17,9 +17,6 @@ class RPCNestedTests : public QObject private Q_SLOTS: void rpcNestedTests(); - -private: - CCoinsViewDB *pcoinsdbview; }; #endif // BITCOIN_QT_TEST_RPC_NESTED_TESTS_H diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 4c04e67ccc..2e11b4f4d6 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -3,18 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparams.h" -#include "rpcnestedtests.h" -#include "util.h" -#include "uritests.h" -#include "compattests.h" +#include <chainparams.h> +#include <qt/test/rpcnestedtests.h> +#include <util.h> +#include <qt/test/uritests.h> +#include <qt/test/compattests.h> #ifdef ENABLE_WALLET -#include "paymentservertests.h" -#include "wallettests.h" +#include <qt/test/paymentservertests.h> +#include <qt/test/wallettests.h> #endif #include <QApplication> diff --git a/src/qt/test/uritests.cpp b/src/qt/test/uritests.cpp index 8b53c0d5c7..e47aa8b546 100644 --- a/src/qt/test/uritests.cpp +++ b/src/qt/test/uritests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "uritests.h" +#include <qt/test/uritests.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> #include <QUrl> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index eeae58bd05..4b7c3bd726 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,22 +1,22 @@ -#include "wallettests.h" +#include <qt/test/wallettests.h> -#include "qt/bitcoinamountfield.h" -#include "qt/callback.h" -#include "qt/optionsmodel.h" -#include "qt/platformstyle.h" -#include "qt/qvalidatedlineedit.h" -#include "qt/sendcoinsdialog.h" -#include "qt/sendcoinsentry.h" -#include "qt/transactiontablemodel.h" -#include "qt/transactionview.h" -#include "qt/walletmodel.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "wallet/wallet.h" -#include "qt/overviewpage.h" -#include "qt/receivecoinsdialog.h" -#include "qt/recentrequeststablemodel.h" -#include "qt/receiverequestdialog.h" +#include <qt/bitcoinamountfield.h> +#include <qt/callback.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/qvalidatedlineedit.h> +#include <qt/sendcoinsdialog.h> +#include <qt/sendcoinsentry.h> +#include <qt/transactiontablemodel.h> +#include <qt/transactionview.h> +#include <qt/walletmodel.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <wallet/wallet.h> +#include <qt/overviewpage.h> +#include <qt/receivecoinsdialog.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/receiverequestdialog.h> #include <QAbstractButton> #include <QAction> @@ -164,7 +164,10 @@ void TestGUI() wallet.SetAddressBook(test.coinbaseKey.GetPubKey().GetID(), "", "receive"); wallet.AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); } - wallet.ScanForWalletTransactions(chainActive.Genesis(), true); + { + LOCK(cs_main); + wallet.ScanForWalletTransactions(chainActive.Genesis(), nullptr, true); + } wallet.SetBroadcastTransactions(true); // Create widgets for sending coins and listing transactions. diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 5bb863451f..b5e4ccb406 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "trafficgraphwidget.h" -#include "clientmodel.h" +#include <qt/trafficgraphwidget.h> +#include <qt/clientmodel.h> #include <QPainter> #include <QColor> diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 74f5c774a0..3f245e6c33 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -2,21 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiondesc.h" - -#include "bitcoinunits.h" -#include "guiutil.h" -#include "paymentserver.h" -#include "transactionrecord.h" - -#include "base58.h" -#include "consensus/consensus.h" -#include "validation.h" -#include "script/script.h" -#include "timedata.h" -#include "util.h" -#include "wallet/db.h" -#include "wallet/wallet.h" +#include <qt/transactiondesc.h> + +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/paymentserver.h> +#include <qt/transactionrecord.h> + +#include <base58.h> +#include <consensus/consensus.h> +#include <validation.h> +#include <script/script.h> +#include <timedata.h> +#include <util.h> +#include <wallet/db.h> +#include <wallet/wallet.h> #include <stdint.h> #include <string> @@ -24,7 +24,7 @@ QString TransactionDesc::FormatTxStatus(const CWalletTx& wtx) { AssertLockHeld(cs_main); - if (!CheckFinalTx(wtx)) + if (!CheckFinalTx(*wtx.tx)) { if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) return tr("Open for %n more block(s)", "", wtx.tx->nLockTime - chainActive.Height()); diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index 65f163deb2..74d7a8f525 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiondescdialog.h" -#include "ui_transactiondescdialog.h" +#include <qt/transactiondescdialog.h> +#include <qt/forms/ui_transactiondescdialog.h> -#include "transactiontablemodel.h" +#include <qt/transactiontablemodel.h> #include <QModelIndex> diff --git a/src/qt/transactionfilterproxy.cpp b/src/qt/transactionfilterproxy.cpp index 44e4ef8238..47890fc7e2 100644 --- a/src/qt/transactionfilterproxy.cpp +++ b/src/qt/transactionfilterproxy.cpp @@ -2,15 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionfilterproxy.h" +#include <qt/transactionfilterproxy.h> -#include "transactiontablemodel.h" -#include "transactionrecord.h" +#include <qt/transactiontablemodel.h> +#include <qt/transactionrecord.h> #include <cstdlib> -#include <QDateTime> - // Earliest date that can be represented (far in the past) const QDateTime TransactionFilterProxy::MIN_DATE = QDateTime::fromTime_t(0); // Last date that can be represented (far in the future) @@ -20,7 +18,7 @@ TransactionFilterProxy::TransactionFilterProxy(QObject *parent) : QSortFilterProxyModel(parent), dateFrom(MIN_DATE), dateTo(MAX_DATE), - addrPrefix(), + m_search_string(), typeFilter(ALL_TYPES), watchOnlyFilter(WatchOnlyFilter_All), minAmount(0), @@ -38,6 +36,7 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & bool involvesWatchAddress = index.data(TransactionTableModel::WatchonlyRole).toBool(); QString address = index.data(TransactionTableModel::AddressRole).toString(); QString label = index.data(TransactionTableModel::LabelRole).toString(); + QString txid = index.data(TransactionTableModel::TxIDRole).toString(); qint64 amount = llabs(index.data(TransactionTableModel::AmountRole).toLongLong()); int status = index.data(TransactionTableModel::StatusRole).toInt(); @@ -51,8 +50,11 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & return false; if(datetime < dateFrom || datetime > dateTo) return false; - if (!address.contains(addrPrefix, Qt::CaseInsensitive) && !label.contains(addrPrefix, Qt::CaseInsensitive)) + if (!address.contains(m_search_string, Qt::CaseInsensitive) && + ! label.contains(m_search_string, Qt::CaseInsensitive) && + ! txid.contains(m_search_string, Qt::CaseInsensitive)) { return false; + } if(amount < minAmount) return false; @@ -66,9 +68,10 @@ void TransactionFilterProxy::setDateRange(const QDateTime &from, const QDateTime invalidateFilter(); } -void TransactionFilterProxy::setAddressPrefix(const QString &_addrPrefix) +void TransactionFilterProxy::setSearchString(const QString &search_string) { - this->addrPrefix = _addrPrefix; + if (m_search_string == search_string) return; + m_search_string = search_string; invalidateFilter(); } diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 7db02cd61f..fea7502b26 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONFILTERPROXY_H #define BITCOIN_QT_TRANSACTIONFILTERPROXY_H -#include "amount.h" +#include <amount.h> #include <QDateTime> #include <QSortFilterProxyModel> @@ -35,7 +35,7 @@ public: }; void setDateRange(const QDateTime &from, const QDateTime &to); - void setAddressPrefix(const QString &addrPrefix); + void setSearchString(const QString &); /** @note Type filter takes a bit field created with TYPE() or ALL_TYPES */ @@ -57,7 +57,7 @@ protected: private: QDateTime dateFrom; QDateTime dateTo; - QString addrPrefix; + QString m_search_string; quint32 typeFilter; WatchOnlyFilter watchOnlyFilter; CAmount minAmount; diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index d40ffd22cd..e3e8377cb9 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionrecord.h" +#include <qt/transactionrecord.h> -#include "base58.h" -#include "consensus/consensus.h" -#include "validation.h" -#include "timedata.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <consensus/consensus.h> +#include <validation.h> +#include <timedata.h> +#include <wallet/wallet.h> #include <stdint.h> @@ -182,7 +182,7 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.depth = wtx.GetDepthInMainChain(); status.cur_num_blocks = chainActive.Height(); - if (!CheckFinalTx(wtx)) + if (!CheckFinalTx(*wtx.tx)) { if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) { diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index a26e676142..ee85acf4ac 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_QT_TRANSACTIONRECORD_H #define BITCOIN_QT_TRANSACTIONRECORD_H -#include "amount.h" -#include "uint256.h" +#include <amount.h> +#include <uint256.h> #include <QList> #include <QString> diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 59cef555b1..67580f11ae 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -2,23 +2,23 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiontablemodel.h" - -#include "addresstablemodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "transactiondesc.h" -#include "transactionrecord.h" -#include "walletmodel.h" - -#include "core_io.h" -#include "validation.h" -#include "sync.h" -#include "uint256.h" -#include "util.h" -#include "wallet/wallet.h" +#include <qt/transactiontablemodel.h> + +#include <qt/addresstablemodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/transactiondesc.h> +#include <qt/transactionrecord.h> +#include <qt/walletmodel.h> + +#include <core_io.h> +#include <validation.h> +#include <sync.h> +#include <uint256.h> +#include <util.h> +#include <wallet/wallet.h> #include <QColor> #include <QDateTime> @@ -80,10 +80,10 @@ public: cachedWallet.clear(); { LOCK2(cs_main, wallet->cs_wallet); - for(std::map<uint256, CWalletTx>::iterator it = wallet->mapWallet.begin(); it != wallet->mapWallet.end(); ++it) + for (const auto& entry : wallet->mapWallet) { - if(TransactionRecord::showTransaction(it->second)) - cachedWallet.append(TransactionRecord::decomposeTransaction(wallet, it->second)); + if (TransactionRecord::showTransaction(entry.second)) + cachedWallet.append(TransactionRecord::decomposeTransaction(wallet, entry.second)); } } } @@ -230,7 +230,7 @@ public: std::map<uint256, CWalletTx>::iterator mi = wallet->mapWallet.find(rec->hash); if(mi != wallet->mapWallet.end()) { - std::string strHex = EncodeHexTx(static_cast<CTransaction>(mi->second)); + std::string strHex = EncodeHexTx(*mi->second.tx); return QString::fromStdString(strHex); } return QString(); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index b1f81498b2..a769707163 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONTABLEMODEL_H #define BITCOIN_QT_TRANSACTIONTABLEMODEL_H -#include "bitcoinunits.h" +#include <qt/bitcoinunits.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 39dfdb587c..b7e39e05ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -2,23 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionview.h" - -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "csvmodelwriter.h" -#include "editaddressdialog.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "sendcoinsdialog.h" -#include "transactiondescdialog.h" -#include "transactionfilterproxy.h" -#include "transactionrecord.h" -#include "transactiontablemodel.h" -#include "walletmodel.h" - -#include "ui_interface.h" +#include <qt/transactionview.h> + +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/csvmodelwriter.h> +#include <qt/editaddressdialog.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/sendcoinsdialog.h> +#include <qt/transactiondescdialog.h> +#include <qt/transactionfilterproxy.h> +#include <qt/transactionrecord.h> +#include <qt/transactiontablemodel.h> +#include <qt/walletmodel.h> + +#include <ui_interface.h> #include <QComboBox> #include <QDateTimeEdit> @@ -95,11 +94,11 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa hlayout->addWidget(typeWidget); - addressWidget = new QLineEdit(this); + search_widget = new QLineEdit(this); #if QT_VERSION >= 0x040700 - addressWidget->setPlaceholderText(tr("Enter address or label to search")); + search_widget->setPlaceholderText(tr("Enter address, transaction id, or label to search")); #endif - hlayout->addWidget(addressWidget); + hlayout->addWidget(search_widget); amountWidget = new QLineEdit(this); #if QT_VERSION >= 0x040700 @@ -187,8 +186,8 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(watchOnlyWidget, SIGNAL(activated(int)), this, SLOT(chooseWatchonly(int))); connect(amountWidget, SIGNAL(textChanged(QString)), amount_typing_delay, SLOT(start())); connect(amount_typing_delay, SIGNAL(timeout()), this, SLOT(changedAmount())); - connect(addressWidget, SIGNAL(textChanged(QString)), prefix_typing_delay, SLOT(start())); - connect(prefix_typing_delay, SIGNAL(timeout()), this, SLOT(changedPrefix())); + connect(search_widget, SIGNAL(textChanged(QString)), prefix_typing_delay, SLOT(start())); + connect(prefix_typing_delay, SIGNAL(timeout()), this, SLOT(changedSearch())); connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); @@ -326,11 +325,11 @@ void TransactionView::chooseWatchonly(int idx) (TransactionFilterProxy::WatchOnlyFilter)watchOnlyWidget->itemData(idx).toInt()); } -void TransactionView::changedPrefix() +void TransactionView::changedSearch() { if(!transactionProxyModel) return; - transactionProxyModel->setAddressPrefix(addressWidget->text()); + transactionProxyModel->setSearchString(search_widget->text()); } void TransactionView::changedAmount() diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 5b4cfd4a88..f72a828b00 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONVIEW_H #define BITCOIN_QT_TRANSACTIONVIEW_H -#include "guiutil.h" +#include <qt/guiutil.h> #include <QWidget> #include <QKeyEvent> @@ -66,7 +66,7 @@ private: QComboBox *dateWidget; QComboBox *typeWidget; QComboBox *watchOnlyWidget; - QLineEdit *addressWidget; + QLineEdit *search_widget; QLineEdit *amountWidget; QMenu *contextMenu; @@ -113,7 +113,7 @@ public Q_SLOTS: void chooseType(int idx); void chooseWatchonly(int idx); void changedAmount(); - void changedPrefix(); + void changedSearch(); void exportClicked(); void focusTransaction(const QModelIndex&); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 5d8c23d13c..6d5a2f3101 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -3,23 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "utilitydialog.h" +#include <qt/utilitydialog.h> -#include "ui_helpmessagedialog.h" +#include <qt/forms/ui_helpmessagedialog.h> -#include "bitcoingui.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "intro.h" -#include "paymentrequestplus.h" -#include "guiutil.h" +#include <qt/bitcoingui.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/intro.h> +#include <qt/paymentrequestplus.h> +#include <qt/guiutil.h> -#include "clientversion.h" -#include "init.h" -#include "util.h" +#include <clientversion.h> +#include <init.h> +#include <util.h> #include <stdio.h> diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 714a594318..573b3f1412 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletframe.h" +#include <qt/walletframe.h> -#include "bitcoingui.h" -#include "walletview.h" +#include <qt/bitcoingui.h> +#include <qt/walletview.h> #include <cassert> #include <cstdio> diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 53b1c2967c..e0193ed475 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -2,32 +2,32 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletmodel.h" - -#include "addresstablemodel.h" -#include "consensus/validation.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "paymentserver.h" -#include "recentrequeststablemodel.h" -#include "sendcoinsdialog.h" -#include "transactiontablemodel.h" - -#include "base58.h" -#include "chain.h" -#include "keystore.h" -#include "validation.h" -#include "net.h" // for g_connman -#include "policy/fees.h" -#include "policy/rbf.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" // for GetBoolArg -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" // for BackupWallet +#include <qt/walletmodel.h> + +#include <qt/addresstablemodel.h> +#include <consensus/validation.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/paymentserver.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/sendcoinsdialog.h> +#include <qt/transactiontablemodel.h> + +#include <base58.h> +#include <chain.h> +#include <keystore.h> +#include <validation.h> +#include <net.h> // for g_connman +#include <policy/fees.h> +#include <policy/rbf.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> // for GetBoolArg +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> // for BackupWallet #include <stdint.h> @@ -659,45 +659,39 @@ bool WalletModel::abandonTransaction(uint256 hash) const bool WalletModel::transactionCanBeBumped(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); + return feebumper::TransactionCanBeBumped(wallet, hash); } bool WalletModel::bumpFee(uint256 hash) { - std::unique_ptr<CFeeBumper> feeBump; - { - CCoinControl coin_control; - coin_control.signalRbf = true; - LOCK2(cs_main, wallet->cs_wallet); - feeBump.reset(new CFeeBumper(wallet, hash, coin_control, 0)); - } - if (feeBump->getResult() != BumpFeeResult::OK) - { + CCoinControl coin_control; + coin_control.signalRbf = true; + std::vector<std::string> errors; + CAmount old_fee; + CAmount new_fee; + CMutableTransaction mtx; + if (feebumper::CreateTransaction(wallet, hash, coin_control, 0 /* totalFee */, errors, old_fee, new_fee, mtx) != feebumper::Result::OK) { QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + - (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + (errors.size() ? QString::fromStdString(errors[0]) : "") +")"); return false; } // allow a user based fee verification QString questionString = tr("Do you want to increase the fee?"); questionString.append("<br />"); - CAmount oldFee = feeBump->getOldFee(); - CAmount newFee = feeBump->getNewFee(); questionString.append("<table style=\"text-align: left;\">"); questionString.append("<tr><td>"); questionString.append(tr("Current fee:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), old_fee)); questionString.append("</td></tr><tr><td>"); questionString.append(tr("Increase:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee - old_fee)); questionString.append("</td></tr><tr><td>"); questionString.append(tr("New fee:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee)); questionString.append("</td></tr></table>"); SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); confirmationDialog.exec(); @@ -715,23 +709,15 @@ bool WalletModel::bumpFee(uint256 hash) } // sign bumped transaction - bool res = false; - { - LOCK2(cs_main, wallet->cs_wallet); - res = feeBump->signTransaction(wallet); - } - if (!res) { + if (!feebumper::SignTransaction(wallet, mtx)) { QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); return false; } // commit the bumped transaction - { - LOCK2(cs_main, wallet->cs_wallet); - res = feeBump->commit(wallet); - } - if(!res) { + uint256 txid; + if (feebumper::CommitTransaction(wallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + - QString::fromStdString(feeBump->getErrors()[0])+")"); + QString::fromStdString(errors[0])+")"); return false; } return true; @@ -751,8 +737,3 @@ int WalletModel::getDefaultConfirmTarget() const { return nTxConfirmTarget; } - -bool WalletModel::getDefaultWalletRbf() const -{ - return fWalletRbf; -} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 05733f8272..8009e54b19 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_QT_WALLETMODEL_H #define BITCOIN_QT_WALLETMODEL_H -#include "paymentrequestplus.h" -#include "walletmodeltransaction.h" +#include <qt/paymentrequestplus.h> +#include <qt/walletmodeltransaction.h> -#include "support/allocators/secure.h" +#include <support/allocators/secure.h> #include <map> #include <vector> @@ -216,8 +216,6 @@ public: int getDefaultConfirmTarget() const; - bool getDefaultWalletRbf() const; - private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eae2c27f8a..2188b39b92 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -2,15 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletmodeltransaction.h" +#include <qt/walletmodeltransaction.h> -#include "policy/policy.h" -#include "wallet/wallet.h" +#include <policy/policy.h> +#include <wallet/wallet.h> WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> &_recipients) : recipients(_recipients), walletTransaction(0), - keyChange(0), fee(0) { walletTransaction = new CWalletTx(); @@ -18,7 +17,6 @@ WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> & WalletModelTransaction::~WalletModelTransaction() { - delete keyChange; delete walletTransaction; } @@ -34,7 +32,7 @@ CWalletTx *WalletModelTransaction::getTransaction() const unsigned int WalletModelTransaction::getTransactionSize() { - return (!walletTransaction ? 0 : ::GetVirtualTransactionSize(*walletTransaction)); + return (!walletTransaction ? 0 : ::GetVirtualTransactionSize(*walletTransaction->tx)); } CAmount WalletModelTransaction::getTransactionFee() const @@ -91,10 +89,10 @@ CAmount WalletModelTransaction::getTotalTransactionAmount() const void WalletModelTransaction::newPossibleKeyChange(CWallet *wallet) { - keyChange = new CReserveKey(wallet); + keyChange.reset(new CReserveKey(wallet)); } CReserveKey *WalletModelTransaction::getPossibleKeyChange() { - return keyChange; + return keyChange.get(); } diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index d7ecd7aa8c..519cf9df99 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETMODELTRANSACTION_H #define BITCOIN_QT_WALLETMODELTRANSACTION_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QObject> @@ -40,7 +40,7 @@ public: private: QList<SendCoinsRecipient> recipients; CWalletTx *walletTransaction; - CReserveKey *keyChange; + std::unique_ptr<CReserveKey> keyChange; CAmount fee; }; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 971f5e0e1a..a589fad9bd 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -2,24 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletview.h" - -#include "addressbookpage.h" -#include "askpassphrasedialog.h" -#include "bitcoingui.h" -#include "clientmodel.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "overviewpage.h" -#include "platformstyle.h" -#include "receivecoinsdialog.h" -#include "sendcoinsdialog.h" -#include "signverifymessagedialog.h" -#include "transactiontablemodel.h" -#include "transactionview.h" -#include "walletmodel.h" - -#include "ui_interface.h" +#include <qt/walletview.h> + +#include <qt/addressbookpage.h> +#include <qt/askpassphrasedialog.h> +#include <qt/bitcoingui.h> +#include <qt/clientmodel.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/overviewpage.h> +#include <qt/platformstyle.h> +#include <qt/receivecoinsdialog.h> +#include <qt/sendcoinsdialog.h> +#include <qt/signverifymessagedialog.h> +#include <qt/transactiontablemodel.h> +#include <qt/transactionview.h> +#include <qt/walletmodel.h> + +#include <ui_interface.h> #include <QAction> #include <QActionGroup> @@ -122,8 +122,8 @@ void WalletView::setWalletModel(WalletModel *_walletModel) overviewPage->setWalletModel(_walletModel); receiveCoinsPage->setModel(_walletModel); sendCoinsPage->setModel(_walletModel); - usedReceivingAddressesPage->setModel(_walletModel->getAddressTableModel()); - usedSendingAddressesPage->setModel(_walletModel->getAddressTableModel()); + usedReceivingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); + usedSendingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); if (_walletModel) { diff --git a/src/qt/walletview.h b/src/qt/walletview.h index c1f8422f0c..4c90874135 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETVIEW_H #define BITCOIN_QT_WALLETVIEW_H -#include "amount.h" +#include <amount.h> #include <QStackedWidget> diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index d78d9a2358..aa6c273431 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "winshutdownmonitor.h" +#include <qt/winshutdownmonitor.h> #if defined(Q_OS_WIN) && QT_VERSION >= 0x050000 -#include "init.h" -#include "util.h" +#include <init.h> +#include <util.h> #include <windows.h> diff --git a/src/random.cpp b/src/random.cpp index 7e0e94439e..3075a77182 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -3,16 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" +#include <random.h> -#include "crypto/sha512.h" -#include "support/cleanse.h" +#include <crypto/sha512.h> +#include <support/cleanse.h> #ifdef WIN32 -#include "compat.h" // for Windows API +#include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include "util.h" // for LogPrint() -#include "utilstrencodings.h" // for GetTime() +#include <util.h> // for LogPrint() +#include <utilstrencodings.h> // for GetTime() #include <stdlib.h> #include <limits> diff --git a/src/random.h b/src/random.h index c60ab36179..a984545467 100644 --- a/src/random.h +++ b/src/random.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H -#include "crypto/chacha20.h" -#include "crypto/common.h" -#include "uint256.h" +#include <crypto/chacha20.h> +#include <crypto/common.h> +#include <uint256.h> #include <stdint.h> @@ -128,7 +128,7 @@ public: * sure that the underlying OS APIs for all platforms support the number. * (many cap out at 256 bytes). */ -static const ssize_t NUM_OS_RANDOM_BYTES = 32; +static const int NUM_OS_RANDOM_BYTES = 32; /** Get 32 bytes of system entropy. Do not use this in application code: use * GetStrongRandBytes instead. diff --git a/src/rest.cpp b/src/rest.cpp index 0b2c843d5f..86ee7804a3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -3,20 +3,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "core_io.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "validation.h" -#include "httpserver.h" -#include "rpc/blockchain.h" -#include "rpc/server.h" -#include "streams.h" -#include "sync.h" -#include "txmempool.h" -#include "utilstrencodings.h" -#include "version.h" +#include <chain.h> +#include <chainparams.h> +#include <core_io.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <validation.h> +#include <httpserver.h> +#include <rpc/blockchain.h> +#include <rpc/server.h> +#include <streams.h> +#include <sync.h> +#include <txmempool.h> +#include <utilstrencodings.h> +#include <version.h> #include <boost/algorithm/string.hpp> @@ -178,8 +178,11 @@ static bool rest_headers(HTTPRequest* req, } case RF_JSON: { UniValue jsonHeaders(UniValue::VARR); - for (const CBlockIndex *pindex : headers) { - jsonHeaders.push_back(blockheaderToJSON(pindex)); + { + LOCK(cs_main); + for (const CBlockIndex *pindex : headers) { + jsonHeaders.push_back(blockheaderToJSON(pindex)); + } } std::string strJSON = jsonHeaders.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -239,7 +242,11 @@ static bool rest_block(HTTPRequest* req, } case RF_JSON: { - UniValue objBlock = blockToJSON(block, pblockindex, showTxDetails); + UniValue objBlock; + { + LOCK(cs_main); + objBlock = blockToJSON(block, pblockindex, showTxDetails); + } std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -409,10 +416,8 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) if (uriParts.size() > 0) { - //inputs is sent over URI scheme (/rest/getutxos/checkmempool/txid1-n/txid2-n/...) - if (uriParts.size() > 0 && uriParts[0] == "checkmempool") - fCheckMemPool = true; + if (uriParts[0] == "checkmempool") fCheckMemPool = true; for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 874645f653..15d4c27750 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -3,28 +3,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/blockchain.h" - -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "coins.h" -#include "consensus/validation.h" -#include "validation.h" -#include "core_io.h" -#include "policy/feerate.h" -#include "policy/policy.h" -#include "primitives/transaction.h" -#include "rpc/server.h" -#include "streams.h" -#include "sync.h" -#include "txdb.h" -#include "txmempool.h" -#include "util.h" -#include "utilstrencodings.h" -#include "hash.h" -#include "warnings.h" +#include <rpc/blockchain.h> + +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <coins.h> +#include <consensus/validation.h> +#include <validation.h> +#include <core_io.h> +#include <policy/feerate.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <streams.h> +#include <sync.h> +#include <txdb.h> +#include <txmempool.h> +#include <util.h> +#include <utilstrencodings.h> +#include <hash.h> +#include <warnings.h> #include <stdint.h> @@ -78,6 +78,7 @@ double GetDifficulty(const CBlockIndex* blockindex) UniValue blockheaderToJSON(const CBlockIndex* blockindex) { + AssertLockHeld(cs_main); UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; @@ -106,6 +107,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) { + AssertLockHeld(cs_main); UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; @@ -926,7 +928,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsdbview, stats)) { + if (GetUTXOStats(pcoinsdbview.get(), stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); @@ -994,7 +996,7 @@ UniValue gettxout(const JSONRPCRequest& request) Coin coin; if (fMempool) { LOCK(mempool.cs); - CCoinsViewMemPool view(pcoinsTip, mempool); + CCoinsViewMemPool view(pcoinsTip.get(), mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { return NullUniValue; } @@ -1046,7 +1048,7 @@ UniValue verifychain(const JSONRPCRequest& request) if (!request.params[1].isNull()) nCheckDepth = request.params[1].get_int(); - return CVerifyDB().VerifyDB(Params(), pcoinsTip, nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), pcoinsTip.get(), nCheckLevel, nCheckDepth); } /** Implementation of IsSuperMajority with better feedback */ @@ -1111,13 +1113,13 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse return rv; } -void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const std::string &name, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) +void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) { // Deployments with timeout value of 0 are hidden. // A timeout value of 0 guarantees a softfork will never be activated. // This is used when softfork codes are merged without specifying the deployment schedule. if (consensusParams.vDeployments[id].nTimeout > 0) - bip9_softforks.push_back(Pair(name, BIP9SoftForkDesc(consensusParams, id))); + bip9_softforks.push_back(Pair(VersionBitsDeploymentInfo[id].name, BIP9SoftForkDesc(consensusParams, id))); } UniValue getblockchaininfo(const JSONRPCRequest& request) @@ -1128,45 +1130,46 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" - " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" - " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" - " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" - " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" + " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" + " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" + " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" + " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" - " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" - " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" - " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" - " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" - " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" + " \"initialblockdownload\": xxxx, (bool) (debug information) estimate of whether this node is in Initial Block Download mode.\n" + " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" + " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" + " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" + " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" + " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" " \"prune_target_size\": xxxxxx, (numeric) the target size used by pruning (only present if automatic pruning is enabled)\n" - " \"softforks\": [ (array) status of softforks in progress\n" + " \"softforks\": [ (array) status of softforks in progress\n" " {\n" - " \"id\": \"xxxx\", (string) name of softfork\n" - " \"version\": xx, (numeric) block version\n" - " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" - " \"status\": xx, (boolean) true if threshold reached\n" + " \"id\": \"xxxx\", (string) name of softfork\n" + " \"version\": xx, (numeric) block version\n" + " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" + " \"status\": xx, (boolean) true if threshold reached\n" " },\n" " }, ...\n" " ],\n" - " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" - " \"xxxx\" : { (string) name of the softfork\n" - " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" - " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" - " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" - " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx, (numeric) height of the first block to which the status applies\n" - " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" - " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" - " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" - " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" - " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" - " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" + " \"xxxx\" : { (string) name of the softfork\n" + " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" + " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" + " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" + " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" " }\n" " }\n" " }\n" - " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" + " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" "}\n" "\nExamples:\n" + HelpExampleCli("getblockchaininfo", "") @@ -1183,6 +1186,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast())); obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip()))); + obj.push_back(Pair("initialblockdownload", IsInitialBlockDownload())); obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); obj.push_back(Pair("size_on_disk", CalculateCurrentUsage())); obj.push_back(Pair("pruned", fPruneMode)); @@ -1210,8 +1214,9 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) softforks.push_back(SoftForkDesc("bip34", 2, tip, consensusParams)); softforks.push_back(SoftForkDesc("bip66", 3, tip, consensusParams)); softforks.push_back(SoftForkDesc("bip65", 4, tip, consensusParams)); - BIP9SoftForkDescPushBack(bip9_softforks, "csv", consensusParams, Consensus::DEPLOYMENT_CSV); - BIP9SoftForkDescPushBack(bip9_softforks, "segwit", consensusParams, Consensus::DEPLOYMENT_SEGWIT); + for (int pos = Consensus::DEPLOYMENT_CSV; pos != Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++pos) { + BIP9SoftForkDescPushBack(bip9_softforks, consensusParams, static_cast<Consensus::DeploymentPos>(pos)); + } obj.push_back(Pair("softforks", softforks)); obj.push_back(Pair("bip9_softforks", bip9_softforks)); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f54f24e2a7..e641493e1e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -3,15 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/client.h" -#include "rpc/protocol.h" -#include "util.h" +#include <rpc/client.h> +#include <rpc/protocol.h> +#include <util.h> #include <set> #include <stdint.h> -#include <univalue.h> - class CRPCConvertParam { public: @@ -93,11 +91,13 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createrawtransaction", 1, "outputs" }, { "createrawtransaction", 2, "locktime" }, { "createrawtransaction", 3, "replaceable" }, + { "decoderawtransaction", 1, "iswitness" }, { "signrawtransaction", 1, "prevtxs" }, { "signrawtransaction", 2, "privkeys" }, { "sendrawtransaction", 1, "allowhighfees" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, + { "fundrawtransaction", 2, "iswitness" }, { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, @@ -141,6 +141,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "echojson", 7, "arg7" }, { "echojson", 8, "arg8" }, { "echojson", 9, "arg9" }, + { "rescanblockchain", 0, "start_height"}, + { "rescanblockchain", 1, "stop_height"}, }; class CRPCConvertTable diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index f79439f038..d003be4954 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -3,34 +3,32 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "consensus/consensus.h" -#include "consensus/params.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "init.h" -#include "validation.h" -#include "miner.h" -#include "net.h" -#include "policy/fees.h" -#include "pow.h" -#include "rpc/blockchain.h" -#include "rpc/mining.h" -#include "rpc/server.h" -#include "txmempool.h" -#include "util.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "warnings.h" +#include <base58.h> +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/params.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <init.h> +#include <validation.h> +#include <miner.h> +#include <net.h> +#include <policy/fees.h> +#include <pow.h> +#include <rpc/blockchain.h> +#include <rpc/mining.h> +#include <rpc/server.h> +#include <txmempool.h> +#include <util.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <warnings.h> #include <memory> #include <stdint.h> -#include <univalue.h> - unsigned int ParseConfirmTarget(const UniValue& value) { int target = value.get_int(); @@ -455,7 +453,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) { // Wait to respond until either the best block changes, OR a minute has passed and there are more transactions uint256 hashWatchedChain; - boost::system_time checktxtime; + std::chrono::steady_clock::time_point checktxtime; unsigned int nTransactionsUpdatedLastLP; if (lpval.isStr()) @@ -476,17 +474,17 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // Release the wallet and main lock while waiting LEAVE_CRITICAL_SECTION(cs_main); { - checktxtime = boost::get_system_time() + boost::posix_time::minutes(1); + checktxtime = std::chrono::steady_clock::now() + std::chrono::minutes(1); - boost::unique_lock<boost::mutex> lock(csBestBlock); + WaitableLock lock(csBestBlock); while (chainActive.Tip()->GetBlockHash() == hashWatchedChain && IsRPCRunning()) { - if (!cvBlockChange.timed_wait(lock, checktxtime)) + if (cvBlockChange.wait_until(lock, checktxtime) == std::cv_status::timeout) { // Timeout: Check transactions for update if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) break; - checktxtime += boost::posix_time::seconds(10); + checktxtime += std::chrono::seconds(10); } } } diff --git a/src/rpc/mining.h b/src/rpc/mining.h index 868d7002b5..8d46273159 100644 --- a/src/rpc/mining.h +++ b/src/rpc/mining.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_RPC_MINING_H #define BITCOIN_RPC_MINING_H -#include "script/script.h" +#include <script/script.h> #include <univalue.h> diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 521b49e2a7..327af2e237 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -3,27 +3,27 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "clientversion.h" -#include "core_io.h" -#include "crypto/ripemd160.h" -#include "init.h" -#include "validation.h" -#include "httpserver.h" -#include "net.h" -#include "netbase.h" -#include "rpc/blockchain.h" -#include "rpc/server.h" -#include "timedata.h" -#include "util.h" -#include "utilstrencodings.h" +#include <base58.h> +#include <chain.h> +#include <clientversion.h> +#include <core_io.h> +#include <crypto/ripemd160.h> +#include <init.h> +#include <validation.h> +#include <httpserver.h> +#include <net.h> +#include <netbase.h> +#include <rpc/blockchain.h> +#include <rpc/server.h> +#include <timedata.h> +#include <util.h> +#include <utilstrencodings.h> #ifdef ENABLE_WALLET -#include "wallet/rpcwallet.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> #endif -#include "warnings.h" +#include <warnings.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -187,17 +187,24 @@ UniValue validateaddress(const JSONRPCRequest& request) ret.push_back(Pair("account", pwallet->mapAddressBook[dest].name)); } if (pwallet) { - const auto& meta = pwallet->mapKeyMetadata; - const CKeyID *keyID = boost::get<CKeyID>(&dest); - auto it = keyID ? meta.find(*keyID) : meta.end(); - if (it == meta.end()) { - it = meta.find(CScriptID(scriptPubKey)); + const CKeyMetadata* meta = nullptr; + if (const CKeyID* key_id = boost::get<CKeyID>(&dest)) { + auto it = pwallet->mapKeyMetadata.find(*key_id); + if (it != pwallet->mapKeyMetadata.end()) { + meta = &it->second; + } + } + if (!meta) { + auto it = pwallet->m_script_metadata.find(CScriptID(scriptPubKey)); + if (it != pwallet->m_script_metadata.end()) { + meta = &it->second; + } } - if (it != meta.end()) { - ret.push_back(Pair("timestamp", it->second.nCreateTime)); - if (!it->second.hdKeypath.empty()) { - ret.push_back(Pair("hdkeypath", it->second.hdKeypath)); - ret.push_back(Pair("hdmasterkeyid", it->second.hdMasterKeyID.GetHex())); + if (meta) { + ret.push_back(Pair("timestamp", meta->nCreateTime)); + if (!meta->hdKeypath.empty()) { + ret.push_back(Pair("hdkeypath", meta->hdKeypath)); + ret.push_back(Pair("hdmasterkeyid", meta->hdMasterKeyID.GetHex())); } } } @@ -533,6 +540,9 @@ uint32_t getCategoryMask(UniValue cats) { if (!GetLogCategory(&flag, &cat)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); } + if (flag == BCLog::NONE) { + return 0; + } mask |= flag; } return mask; @@ -542,16 +552,32 @@ UniValue logging(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 2) { throw std::runtime_error( - "logging [include,...] <exclude>\n" + "logging ( <include> <exclude> )\n" "Gets and sets the logging configuration.\n" - "When called without an argument, returns the list of categories that are currently being debug logged.\n" - "When called with arguments, adds or removes categories from debug logging.\n" + "When called without an argument, returns the list of categories with status that are currently being debug logged or not.\n" + "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" + "The arguments are evaluated in order \"include\", \"exclude\".\n" + "If an item is both included and excluded, it will thus end up being excluded.\n" "The valid logging categories are: " + ListLogCategories() + "\n" - "libevent logging is configured on startup and cannot be modified by this RPC during runtime." - "Arguments:\n" - "1. \"include\" (array of strings) add debug logging for these categories.\n" - "2. \"exclude\" (array of strings) remove debug logging for these categories.\n" - "\nResult: <categories> (string): a list of the logging categories that are active.\n" + "In addition, the following are available as category names with special meanings:\n" + " - \"all\", \"1\" : represent all logging categories.\n" + " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" + "\nArguments:\n" + "1. \"include\" (array of strings, optional) A json array of categories to add debug logging\n" + " [\n" + " \"category\" (string) the valid logging category\n" + " ,...\n" + " ]\n" + "2. \"exclude\" (array of strings, optional) A json array of categories to remove debug logging\n" + " [\n" + " \"category\" (string) the valid logging category\n" + " ,...\n" + " ]\n" + "\nResult:\n" + "{ (json object where keys are the logging categories, and values indicates its status\n" + " \"category\": 0|1, (numeric) if being debug logged or not. 0:inactive, 1:active\n" + " ...\n" + "}\n" "\nExamples:\n" + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + HelpExampleRpc("logging", "[\"all\"], \"[libevent]\"") @@ -608,6 +634,7 @@ static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- { "control", "getmemoryinfo", &getmemoryinfo, {"mode"} }, + { "control", "logging", &logging, {"include", "exclude"}}, { "util", "validateaddress", &validateaddress, {"address"} }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, {"nrequired","keys"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, @@ -617,7 +644,6 @@ static const CRPCCommand commands[] = { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, { "hidden", "echojson", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "logging", &logging, {"include", "exclude"}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index a3d3df26a3..9de249855c 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -2,24 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" - -#include "chainparams.h" -#include "clientversion.h" -#include "core_io.h" -#include "validation.h" -#include "net.h" -#include "net_processing.h" -#include "netbase.h" -#include "policy/policy.h" -#include "protocol.h" -#include "sync.h" -#include "timedata.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" -#include "warnings.h" +#include <rpc/server.h> + +#include <chainparams.h> +#include <clientversion.h> +#include <core_io.h> +#include <validation.h> +#include <net.h> +#include <net_processing.h> +#include <netbase.h> +#include <policy/policy.h> +#include <rpc/protocol.h> +#include <sync.h> +#include <timedata.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> +#include <warnings.h> #include <univalue.h> @@ -92,7 +92,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Satoshi:0.8.5/\", (string) The string version\n" " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" - " \"addnode\": true|false, (boolean) Whether connection was due to addnode and is using an addnode slot\n" + " \"addnode\": true|false, (boolean) Whether connection was due to addnode/-connect or if it was an automatic/inbound connection\n" " \"startingheight\": n, (numeric) The starting height (block) of the peer\n" " \"banscore\": n, (numeric) The ban score\n" " \"synced_headers\": n, (numeric) The last header we have in common with this peer\n" @@ -156,7 +156,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) // their ver message. obj.push_back(Pair("subver", stats.cleanSubVer)); obj.push_back(Pair("inbound", stats.fInbound)); - obj.push_back(Pair("addnode", stats.fAddnode)); + obj.push_back(Pair("addnode", stats.m_manual_connection)); obj.push_back(Pair("startingheight", stats.nStartingHeight)); if (fStateStats) { obj.push_back(Pair("banscore", statestats.nMisbehavior)); @@ -201,6 +201,8 @@ UniValue addnode(const JSONRPCRequest& request) "addnode \"node\" \"add|remove|onetry\"\n" "\nAttempts to add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" + "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" + "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n" "\nArguments:\n" "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" "2. \"command\" (string, required) 'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once\n" @@ -217,7 +219,7 @@ UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str()); + g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } @@ -569,11 +571,11 @@ UniValue listbanned(const JSONRPCRequest& request) g_connman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); - for (banmap_t::iterator it = banMap.begin(); it != banMap.end(); it++) + for (const auto& entry : banMap) { - CBanEntry banEntry = (*it).second; + const CBanEntry& banEntry = entry.second; UniValue rec(UniValue::VOBJ); - rec.push_back(Pair("address", (*it).first.ToString())); + rec.push_back(Pair("address", entry.first.ToString())); rec.push_back(Pair("banned_until", banEntry.nBanUntil)); rec.push_back(Pair("ban_created", banEntry.nCreateTime)); rec.push_back(Pair("ban_reason", banEntry.banReasonToString())); diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index 1f4ae75b18..d999a08d74 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -3,16 +3,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/protocol.h" +#include <rpc/protocol.h> -#include "random.h" -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" -#include "utiltime.h" -#include "version.h" +#include <random.h> +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> +#include <utiltime.h> +#include <version.h> -#include <stdint.h> #include <fstream> /** diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index cb668f3db9..4b9bd41994 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_RPCPROTOCOL_H #define BITCOIN_RPCPROTOCOL_H -#include "fs.h" +#include <fs.h> #include <list> #include <map> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index d860dbc244..1049131d5e 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -3,33 +3,35 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "coins.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "init.h" -#include "keystore.h" -#include "validation.h" -#include "merkleblock.h" -#include "net.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/transaction.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "script/standard.h" -#include "txmempool.h" -#include "uint256.h" -#include "utilstrencodings.h" +#include <base58.h> +#include <chain.h> +#include <coins.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <init.h> +#include <keystore.h> +#include <validation.h> +#include <validationinterface.h> +#include <merkleblock.h> +#include <net.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <script/standard.h> +#include <txmempool.h> +#include <uint256.h> +#include <utilstrencodings.h> #ifdef ENABLE_WALLET -#include "wallet/rpcwallet.h" -#include "wallet/wallet.h" +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> #endif +#include <future> #include <stdint.h> #include <univalue.h> @@ -62,12 +64,15 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue getrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( - "getrawtransaction \"txid\" ( verbose )\n" + "getrawtransaction \"txid\" ( verbose \"blockhash\" )\n" "\nNOTE: By default this function only works for mempool transactions. If the -txindex option is\n" - "enabled, it also works for blockchain transactions.\n" + "enabled, it also works for blockchain transactions. If the block which contains the transaction\n" + "is known, its hash can be provided even for nodes without -txindex. Note that if a blockhash is\n" + "provided, only that block will be searched and if the transaction is in the mempool or other\n" + "blocks, or if this node does not have the given block available, the transaction will not be found.\n" "DEPRECATED: for now, it also works for transactions with unspent outputs.\n" "\nReturn the raw transaction data.\n" @@ -76,13 +81,15 @@ UniValue getrawtransaction(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (bool, optional, default=false) If false, return a string, otherwise return a json object\n" + "2. verbose (bool, optional, default=false) If false, return a string, otherwise return a json object\n" + "3. \"blockhash\" (string, optional) The block in which to look for the transaction\n" "\nResult (if verbose is not set or set to false):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" "\nResult (if verbose is set to true):\n" "{\n" + " \"in_active_chain\": b, (bool) Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)\n" " \"hex\" : \"data\", (string) The serialized, hex-encoded data for 'txid'\n" " \"txid\" : \"id\", (string) The transaction id (same as provided)\n" " \"hash\" : \"id\", (string) The transaction hash (differs from txid for witness transactions)\n" @@ -130,42 +137,56 @@ UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\"") + HelpExampleCli("getrawtransaction", "\"mytxid\" true") + HelpExampleRpc("getrawtransaction", "\"mytxid\", true") + + HelpExampleCli("getrawtransaction", "\"mytxid\" false \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") ); LOCK(cs_main); + bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); + CBlockIndex* blockindex = nullptr; // Accept either a bool (true) or a num (>=1) to indicate verbose output. bool fVerbose = false; if (!request.params[1].isNull()) { - if (request.params[1].isNum()) { - if (request.params[1].get_int() != 0) { - fVerbose = true; - } - } - else if(request.params[1].isBool()) { - if(request.params[1].isTrue()) { - fVerbose = true; - } - } - else { - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid type provided. Verbose parameter must be a boolean."); + fVerbose = request.params[1].isNum() ? (request.params[1].get_int() != 0) : request.params[1].get_bool(); + } + + if (!request.params[2].isNull()) { + uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); + BlockMap::iterator it = mapBlockIndex.find(blockhash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } + blockindex = it->second; + in_active_chain = chainActive.Contains(blockindex); } CTransactionRef tx; - uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string(fTxIndex ? "No such mempool or blockchain transaction" - : "No such mempool transaction. Use -txindex to enable blockchain transaction queries") + - ". Use gettransaction for wallet transactions."); + uint256 hash_block; + if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) { + std::string errmsg; + if (blockindex) { + if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); + } + errmsg = "No such transaction found in the provided block"; + } else { + errmsg = fTxIndex + ? "No such mempool or blockchain transaction" + : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + } + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); + } - if (!fVerbose) + if (!fVerbose) { return EncodeHexTx(*tx, RPCSerializationFlags()); + } UniValue result(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, result); + if (blockindex) result.push_back(Pair("in_active_chain", in_active_chain)); + TxToJSON(*tx, hash_block, result); return result; } @@ -328,7 +349,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) + HelpExampleRpc("createrawtransaction", "\"[{\\\"txid\\\":\\\"myid\\\",\\\"vout\\\":0}]\", \"{\\\"data\\\":\\\"00010203\\\"}\"") ); - RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VOBJ, UniValue::VNUM}, true); + RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VOBJ, UniValue::VNUM, UniValue::VBOOL}, true); if (request.params[0].isNull() || request.params[1].isNull()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); @@ -420,13 +441,15 @@ UniValue createrawtransaction(const JSONRPCRequest& request) UniValue decoderawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "decoderawtransaction \"hexstring\"\n" + "decoderawtransaction \"hexstring\" ( iswitness )\n" "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n" "\nArguments:\n" "1. \"hexstring\" (string, required) The transaction hex string\n" + "2. iswitness (boolean, optional) Whether the transaction hex is a serialized witness transaction\n" + " If iswitness is not present, heuristic tests will be used in decoding\n" "\nResult:\n" "{\n" @@ -474,12 +497,16 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) ); LOCK(cs_main); - RPCTypeCheck(request.params, {UniValue::VSTR}); + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); CMutableTransaction mtx; - if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) + bool try_witness = request.params[1].isNull() ? true : request.params[1].get_bool(); + bool try_no_witness = request.params[1].isNull() ? true : !request.params[1].get_bool(); + + if (!DecodeHexTx(mtx, request.params[0].get_str(), try_no_witness, try_witness)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + } UniValue result(UniValue::VOBJ); TxToUniv(CTransaction(std::move(mtx)), uint256(), result, false); @@ -917,7 +944,9 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); - LOCK(cs_main); + + std::promise<void> promise; + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); // parse hex string from parameter @@ -931,6 +960,8 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) if (!request.params[1].isNull() && request.params[1].get_bool()) nMaxRawTxFee = 0; + { // cs_main scope + LOCK(cs_main); CCoinsViewCache &view = *pcoinsTip; bool fHaveChain = false; for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { @@ -952,10 +983,28 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } throw JSONRPCError(RPC_TRANSACTION_ERROR, state.GetRejectReason()); } + } else { + // If wallet is enabled, ensure that the wallet has been made aware + // of the new transaction prior to returning. This prevents a race + // where a user might call sendrawtransaction with a transaction + // to/from their wallet, immediately call some wallet RPC, and get + // a stale result because callbacks have not yet been processed. + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); } } else if (fHaveChain) { throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); + } else { + // Make sure we don't block forever if re-sending + // a transaction already in mempool. + promise.set_value(); } + + } // cs_main + + promise.get_future().wait(); + if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); @@ -964,15 +1013,16 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) { pnode->PushInventory(inv); }); + return hashTx.GetHex(); } static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose"} }, + { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose","blockhash"} }, { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring"} }, + { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, diff --git a/src/rpc/safemode.cpp b/src/rpc/safemode.cpp index 24770ad47f..9f3a9d30b8 100644 --- a/src/rpc/safemode.cpp +++ b/src/rpc/safemode.cpp @@ -1,8 +1,8 @@ -#include "safemode.h" +#include <rpc/safemode.h> -#include "rpc/protocol.h" -#include "util.h" -#include "warnings.h" +#include <rpc/protocol.h> +#include <util.h> +#include <warnings.h> void ObserveSafeMode() { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index a73b697e01..2e7e0ba474 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -3,18 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" +#include <rpc/server.h> -#include "base58.h" -#include "fs.h" -#include "init.h" -#include "random.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" - -#include <univalue.h> +#include <base58.h> +#include <fs.h> +#include <init.h> +#include <random.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> @@ -160,8 +158,8 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& std::set<rpcfn_type> setDone; std::vector<std::pair<std::string, const CRPCCommand*> > vCommands; - for (std::map<std::string, const CRPCCommand*>::const_iterator mi = mapCommands.begin(); mi != mapCommands.end(); ++mi) - vCommands.push_back(make_pair(mi->second->category + mi->first, mi->second)); + for (const auto& entry : mapCommands) + vCommands.push_back(make_pair(entry.second->category + entry.first, entry.second)); sort(vCommands.begin(), vCommands.end()); JSONRPCRequest jreq(helpreq); @@ -389,11 +387,10 @@ bool IsDeprecatedRPCEnabled(const std::string& method) return find(enabled_methods.begin(), enabled_methods.end(), method) != enabled_methods.end(); } -static UniValue JSONRPCExecOne(const UniValue& req) +static UniValue JSONRPCExecOne(JSONRPCRequest jreq, const UniValue& req) { UniValue rpc_result(UniValue::VOBJ); - JSONRPCRequest jreq; try { jreq.parse(req); @@ -413,11 +410,11 @@ static UniValue JSONRPCExecOne(const UniValue& req) return rpc_result; } -std::string JSONRPCExecBatch(const UniValue& vReq) +std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) { UniValue ret(UniValue::VARR); for (unsigned int reqIdx = 0; reqIdx < vReq.size(); reqIdx++) - ret.push_back(JSONRPCExecOne(vReq[reqIdx])); + ret.push_back(JSONRPCExecOne(jreq, vReq[reqIdx])); return ret.write() + "\n"; } diff --git a/src/rpc/server.h b/src/rpc/server.h index 31d6304271..cd201bf316 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_RPCSERVER_H #define BITCOIN_RPCSERVER_H -#include "amount.h" -#include "rpc/protocol.h" -#include "uint256.h" +#include <amount.h> +#include <rpc/protocol.h> +#include <uint256.h> #include <list> #include <map> @@ -191,7 +191,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri bool StartRPC(); void InterruptRPC(); void StopRPC(); -std::string JSONRPCExecBatch(const UniValue& vReq); +std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); // Retrieves any serialization flags requested in command line argument int RPCSerializationFlags(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4edb2c6d9b..260f6fa60e 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "scheduler.h" +#include <scheduler.h> -#include "random.h" -#include "reverselock.h" +#include <random.h> +#include <reverselock.h> #include <assert.h> #include <boost/bind.hpp> diff --git a/src/scheduler.h b/src/scheduler.h index db93bcb21e..b99f165343 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -14,7 +14,7 @@ #include <boost/thread.hpp> #include <map> -#include "sync.h" +#include <sync.h> // // Simple class for background tasks that should be run diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 03128917fd..77edc85b8c 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> -#include "primitives/transaction.h" -#include "pubkey.h" -#include "script/interpreter.h" -#include "version.h" +#include <primitives/transaction.h> +#include <pubkey.h> +#include <script/interpreter.h> +#include <version.h> namespace { diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 33bf80e5a7..f09cd59b92 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -9,7 +9,7 @@ #include <stdint.h> #if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #if defined(_WIN32) #if defined(DLL_EXPORT) #if defined(HAVE_FUNC_ATTRIBUTE_DLLEXPORT) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 2f7b8e3a03..62042e8e70 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -3,15 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "interpreter.h" +#include <script/interpreter.h> -#include "primitives/transaction.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "pubkey.h" -#include "script/script.h" -#include "uint256.h" +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <pubkey.h> +#include <script/script.h> +#include <uint256.h> typedef std::vector<unsigned char> valtype; @@ -174,7 +173,13 @@ bool static IsLowDERSignature(const valtype &vchSig, ScriptError* serror) { if (!IsValidSignatureEncoding(vchSig)) { return set_error(serror, SCRIPT_ERR_SIG_DER); } + // https://bitcoin.stackexchange.com/a/12556: + // Also note that inside transaction signatures, an extra hashtype byte + // follows the actual signature data. std::vector<unsigned char> vchSigCopy(vchSig.begin(), vchSig.begin() + vchSig.size() - 1); + // If the S value is above the order of the curve divided by two, its + // complement modulo the order could have been used instead, which is + // one byte shorter when encoded correctly. if (!CPubKey::CheckLowS(vchSigCopy)) { return set_error(serror, SCRIPT_ERR_SIG_HIGH_S); } @@ -349,9 +354,6 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { if (!(flags & SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY)) { // not enabled; treat as a NOP2 - if (flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) { - return set_error(serror, SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS); - } break; } @@ -391,9 +393,6 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { if (!(flags & SCRIPT_VERIFY_CHECKSEQUENCEVERIFY)) { // not enabled; treat as a NOP3 - if (flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) { - return set_error(serror, SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS); - } break; } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 1cb9cc7899..83a96739b1 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_SCRIPT_INTERPRETER_H #define BITCOIN_SCRIPT_INTERPRETER_H -#include "script_error.h" -#include "primitives/transaction.h" +#include <script/script_error.h> +#include <primitives/transaction.h> #include <vector> #include <stdint.h> @@ -27,37 +27,40 @@ enum SIGHASH_ANYONECANPAY = 0x80, }; -/** Script verification flags */ +/** Script verification flags. + * + * All flags are intended to be soft forks: the set of acceptable scripts under + * flags (A | B) is a subset of the acceptable scripts under flag (A). + */ enum { SCRIPT_VERIFY_NONE = 0, - // Evaluate P2SH subscripts (softfork safe, BIP16). + // Evaluate P2SH subscripts (BIP16). SCRIPT_VERIFY_P2SH = (1U << 0), // Passing a non-strict-DER signature or one with undefined hashtype to a checksig operation causes script failure. // Evaluating a pubkey that is not (0x04 + 64 bytes) or (0x02 or 0x03 + 32 bytes) by checksig causes script failure. - // (softfork safe, but not used or intended as a consensus rule). + // (not used or intended as a consensus rule). SCRIPT_VERIFY_STRICTENC = (1U << 1), - // Passing a non-strict-DER signature to a checksig operation causes script failure (softfork safe, BIP62 rule 1) + // Passing a non-strict-DER signature to a checksig operation causes script failure (BIP62 rule 1) SCRIPT_VERIFY_DERSIG = (1U << 2), // Passing a non-strict-DER signature or one with S > order/2 to a checksig operation causes script failure - // (softfork safe, BIP62 rule 5). + // (BIP62 rule 5). SCRIPT_VERIFY_LOW_S = (1U << 3), - // verify dummy stack item consumed by CHECKMULTISIG is of zero-length (softfork safe, BIP62 rule 7). + // verify dummy stack item consumed by CHECKMULTISIG is of zero-length (BIP62 rule 7). SCRIPT_VERIFY_NULLDUMMY = (1U << 4), - // Using a non-push operator in the scriptSig causes script failure (softfork safe, BIP62 rule 2). + // Using a non-push operator in the scriptSig causes script failure (BIP62 rule 2). SCRIPT_VERIFY_SIGPUSHONLY = (1U << 5), // Require minimal encodings for all push operations (OP_0... OP_16, OP_1NEGATE where possible, direct // pushes up to 75 bytes, OP_PUSHDATA up to 255 bytes, OP_PUSHDATA2 for anything larger). Evaluating // any other push causes the script to fail (BIP62 rule 3). // In addition, whenever a stack element is interpreted as a number, it must be of minimal length (BIP62 rule 4). - // (softfork safe) SCRIPT_VERIFY_MINIMALDATA = (1U << 6), // Discourage use of NOPs reserved for upgrades (NOP1-10) @@ -68,12 +71,14 @@ enum // discouraged NOPs fails the script. This verification flag will never be // a mandatory flag applied to scripts in a block. NOPs that are not // executed, e.g. within an unexecuted IF ENDIF block, are *not* rejected. + // NOPs that have associated forks to give them new meaning (CLTV, CSV) + // are not subject to this rule. SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS = (1U << 7), // Require that only a single stack element remains after evaluation. This changes the success criterion from // "At least one stack element must remain, and when interpreted as a boolean, it must be true" to // "Exactly one stack element must remain, and when interpreted as a boolean, it must be true". - // (softfork safe, BIP62 rule 6) + // (BIP62 rule 6) // Note: CLEANSTACK should never be used without P2SH or WITNESS. SCRIPT_VERIFY_CLEANSTACK = (1U << 8), diff --git a/src/script/ismine.cpp b/src/script/ismine.cpp index 6b68f0679e..5849f55651 100644 --- a/src/script/ismine.cpp +++ b/src/script/ismine.cpp @@ -3,13 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "ismine.h" +#include <script/ismine.h> -#include "key.h" -#include "keystore.h" -#include "script/script.h" -#include "script/standard.h" -#include "script/sign.h" +#include <key.h> +#include <keystore.h> +#include <script/script.h> +#include <script/sign.h> typedef std::vector<unsigned char> valtype; diff --git a/src/script/ismine.h b/src/script/ismine.h index 1aa5937b34..77ffa82432 100644 --- a/src/script/ismine.h +++ b/src/script/ismine.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_SCRIPT_ISMINE_H #define BITCOIN_SCRIPT_ISMINE_H -#include "script/standard.h" +#include <script/standard.h> #include <stdint.h> diff --git a/src/script/script.cpp b/src/script/script.cpp index a10b619f7d..a7bfb170eb 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -3,10 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script.h" +#include <script/script.h> -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <tinyformat.h> +#include <utilstrencodings.h> const char* GetOpName(opcodetype opcode) { diff --git a/src/script/script.h b/src/script/script.h index 2a92060543..263db62edf 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_H #define BITCOIN_SCRIPT_SCRIPT_H -#include "crypto/common.h" -#include "prevector.h" -#include "serialize.h" +#include <crypto/common.h> +#include <prevector.h> +#include <serialize.h> #include <assert.h> #include <climits> diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index 6c590f53e3..32aa639ac4 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script_error.h" +#include <script/script_error.h> const char* ScriptErrorString(const ScriptError serror) { diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 4cc7afa2f5..88d1d67559 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -3,15 +3,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sigcache.h" +#include <script/sigcache.h> -#include "memusage.h" -#include "pubkey.h" -#include "random.h" -#include "uint256.h" -#include "util.h" +#include <memusage.h> +#include <pubkey.h> +#include <random.h> +#include <uint256.h> +#include <util.h> -#include "cuckoocache.h" +#include <cuckoocache.h> #include <boost/thread.hpp> namespace { diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 5832b264b3..1019061fbb 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_SCRIPT_SIGCACHE_H #define BITCOIN_SCRIPT_SIGCACHE_H -#include "script/interpreter.h" +#include <script/interpreter.h> #include <vector> diff --git a/src/script/sign.cpp b/src/script/sign.cpp index ac58b690a2..117a4d8a52 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -3,14 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script/sign.h" - -#include "key.h" -#include "keystore.h" -#include "policy/policy.h" -#include "primitives/transaction.h" -#include "script/standard.h" -#include "uint256.h" +#include <script/sign.h> + +#include <key.h> +#include <keystore.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/standard.h> +#include <uint256.h> typedef std::vector<unsigned char> valtype; diff --git a/src/script/sign.h b/src/script/sign.h index a0d8ee4ff9..400c0c0865 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_SCRIPT_SIGN_H #define BITCOIN_SCRIPT_SIGN_H -#include "script/interpreter.h" +#include <script/interpreter.h> class CKeyID; class CKeyStore; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index f57f1f61b4..b7b33fade6 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script/standard.h" +#include <script/standard.h> -#include "pubkey.h" -#include "script/script.h" -#include "util.h" -#include "utilstrencodings.h" +#include <pubkey.h> +#include <script/script.h> +#include <util.h> +#include <utilstrencodings.h> typedef std::vector<unsigned char> valtype; diff --git a/src/script/standard.h b/src/script/standard.h index fa07ea88c1..3eeeabdc15 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_SCRIPT_STANDARD_H #define BITCOIN_SCRIPT_STANDARD_H -#include "script/interpreter.h" -#include "uint256.h" +#include <script/interpreter.h> +#include <uint256.h> #include <boost/variant.hpp> diff --git a/src/serialize.h b/src/serialize.h index eeb05fa76c..02d3e4f7c6 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H -#include "compat/endian.h" +#include <compat/endian.h> #include <algorithm> #include <assert.h> @@ -21,7 +21,7 @@ #include <utility> #include <vector> -#include "prevector.h" +#include <prevector.h> static const unsigned int MAX_SIZE = 0x02000000; @@ -732,8 +732,8 @@ template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m) { WriteCompactSize(os, m.size()); - for (typename std::map<K, T, Pred, A>::const_iterator mi = m.begin(); mi != m.end(); ++mi) - Serialize(os, (*mi)); + for (const auto& entry : m) + Serialize(os, entry); } template<typename Stream, typename K, typename T, typename Pred, typename A> diff --git a/src/streams.h b/src/streams.h index 9a3badea57..c26b1f618c 100644 --- a/src/streams.h +++ b/src/streams.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H -#include "support/allocators/zeroafterfree.h" -#include "serialize.h" +#include <support/allocators/zeroafterfree.h> +#include <serialize.h> #include <algorithm> #include <assert.h> @@ -241,8 +241,8 @@ public: const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } + iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } @@ -345,18 +345,16 @@ public: // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext >= vch.size()) + if (nReadPosNext > vch.size()) { + throw std::ios_base::failure("CDataStream::read(): end of data"); + } + memcpy(pch, &vch[nReadPos], nSize); + if (nReadPosNext == vch.size()) { - if (nReadPosNext > vch.size()) - { - throw std::ios_base::failure("CDataStream::read(): end of data"); - } - memcpy(pch, &vch[nReadPos], nSize); nReadPos = 0; vch.clear(); return; } - memcpy(pch, &vch[nReadPos], nSize); nReadPos = nReadPosNext; } diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 39347c73bb..7cd031bfe8 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H -#include "support/lockedpool.h" -#include "support/cleanse.h" +#include <support/lockedpool.h> +#include <support/cleanse.h> #include <string> diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 618874ceee..23ef96c13f 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H #define BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H -#include "support/cleanse.h" +#include <support/cleanse.h> #include <memory> #include <vector> diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp index 95899c9f02..8d3c7369b4 100644 --- a/src/support/cleanse.cpp +++ b/src/support/cleanse.cpp @@ -3,10 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "cleanse.h" +#include <support/cleanse.h> #include <cstring> +#if defined(_MSC_VER) +#include <Windows.h> // For SecureZeroMemory. +#endif + /* Compilers have a bad habit of removing "superfluous" memset calls that * are trying to zero memory. For example, when memset()ing a buffer and * then free()ing it, the compiler might decide that the memset is @@ -32,7 +36,7 @@ void memory_cleanse(void *ptr, size_t len) might try to eliminate "superfluous" memsets. If there's an easy way to detect memset_s, it would be better to use that. */ #if defined(_MSC_VER) - __asm; + SecureZeroMemory(ptr, len); #else __asm__ __volatile__("" : : "r"(ptr) : "memory"); #endif diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 2ead72185f..98e8694181 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "support/lockedpool.h" -#include "support/cleanse.h" +#include <support/lockedpool.h> +#include <support/cleanse.h> #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #ifdef WIN32 diff --git a/src/sync.cpp b/src/sync.cpp index 87024ccdf2..3f51383ea2 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -2,16 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sync.h" +#include <sync.h> -#include "util.h" -#include "utilstrencodings.h" +#include <set> +#include <util.h> +#include <utilstrencodings.h> #include <stdio.h> -#include <boost/thread.hpp> - #ifdef DEBUG_LOCKCONTENTION +#if !defined(HAVE_THREAD_LOCAL) +static_assert(false, "thread_local is not supported"); +#endif void PrintLockContention(const char* pszName, const char* pszFile, int nLine) { LogPrintf("LOCKCONTENTION: %s\n", pszName); @@ -45,8 +47,8 @@ struct CLockLocation { return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); } - bool fTry; private: + bool fTry; std::string mutexName; std::string sourceFile; int sourceLine; @@ -67,10 +69,10 @@ struct LockData { LockOrders lockorders; InvLockOrders invlockorders; - boost::mutex dd_mutex; + std::mutex dd_mutex; } static lockdata; -boost::thread_specific_ptr<LockStack> lockstack; +static thread_local std::unique_ptr<LockStack> lockstack; static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, const LockStack& s1, const LockStack& s2) { @@ -100,12 +102,12 @@ static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, static void push_lock(void* c, const CLockLocation& locklocation) { - if (lockstack.get() == nullptr) + if (!lockstack) lockstack.reset(new LockStack); - boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - (*lockstack).push_back(std::make_pair(c, locklocation)); + lockstack->push_back(std::make_pair(c, locklocation)); for (const std::pair<void*, CLockLocation> & i : (*lockstack)) { if (i.first == c) @@ -155,13 +157,23 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, abort(); } +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +{ + for (const std::pair<void*, CLockLocation>& i : *lockstack) { + if (i.first == cs) { + fprintf(stderr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + abort(); + } + } +} + void DeleteLock(void* cs) { if (!lockdata.available) { // We're already shutting down. return; } - boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); std::pair<void*, void*> item = std::make_pair(cs, nullptr); LockOrders::iterator it = lockdata.lockorders.lower_bound(item); while (it != lockdata.lockorders.end() && it->first.first == cs) { diff --git a/src/sync.h b/src/sync.h index 0871c5fb4d..3af27c65d0 100644 --- a/src/sync.h +++ b/src/sync.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_SYNC_H #define BITCOIN_SYNC_H -#include "threadsafety.h" +#include <threadsafety.h> -#include <boost/thread/condition_variable.hpp> -#include <boost/thread/mutex.hpp> -#include <boost/thread/recursive_mutex.hpp> +#include <condition_variable> +#include <thread> +#include <mutex> //////////////////////////////////////////////// @@ -21,17 +21,17 @@ /* CCriticalSection mutex; - boost::recursive_mutex mutex; + std::recursive_mutex mutex; LOCK(mutex); - boost::unique_lock<boost::recursive_mutex> criticalblock(mutex); + std::unique_lock<std::recursive_mutex> criticalblock(mutex); LOCK2(mutex1, mutex2); - boost::unique_lock<boost::recursive_mutex> criticalblock1(mutex1); - boost::unique_lock<boost::recursive_mutex> criticalblock2(mutex2); + std::unique_lock<std::recursive_mutex> criticalblock1(mutex1); + std::unique_lock<std::recursive_mutex> criticalblock2(mutex2); TRY_LOCK(mutex, name); - boost::unique_lock<boost::recursive_mutex> name(mutex, boost::try_to_lock_t); + std::unique_lock<std::recursive_mutex> name(mutex, std::try_to_lock_t); ENTER_CRITICAL_SECTION(mutex); // no RAII mutex.lock(); @@ -75,20 +75,23 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void LeaveCritical(); std::string LocksHeld(); void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); void DeleteLock(void* cs); #else void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} +void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} void static inline DeleteLock(void* cs) {} #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) +#define AssertLockNotHeld(cs) AssertLockNotHeldInternal(#cs, __FILE__, __LINE__, &cs) /** - * Wrapped boost mutex: supports recursive locking, but no waiting + * Wrapped mutex: supports recursive locking, but no waiting * TODO: We should move away from using the recursive lock by default. */ -class CCriticalSection : public AnnotatedMixin<boost::recursive_mutex> +class CCriticalSection : public AnnotatedMixin<std::recursive_mutex> { public: ~CCriticalSection() { @@ -96,22 +99,24 @@ public: } }; -/** Wrapped boost mutex: supports waiting but not recursive locking */ -typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection; +/** Wrapped mutex: supports waiting but not recursive locking */ +typedef AnnotatedMixin<std::mutex> CWaitableCriticalSection; -/** Just a typedef for boost::condition_variable, can be wrapped later if desired */ -typedef boost::condition_variable CConditionVariable; +/** Just a typedef for std::condition_variable, can be wrapped later if desired */ +typedef std::condition_variable CConditionVariable; + +/** Just a typedef for std::unique_lock, can be wrapped later if desired */ +typedef std::unique_lock<std::mutex> WaitableLock; #ifdef DEBUG_LOCKCONTENTION void PrintLockContention(const char* pszName, const char* pszFile, int nLine); #endif -/** Wrapper around boost::unique_lock<Mutex> */ -template <typename Mutex> -class SCOPED_LOCKABLE CMutexLock +/** Wrapper around std::unique_lock<CCriticalSection> */ +class SCOPED_LOCKABLE CCriticalBlock { private: - boost::unique_lock<Mutex> lock; + std::unique_lock<CCriticalSection> lock; void Enter(const char* pszName, const char* pszFile, int nLine) { @@ -136,7 +141,7 @@ private: } public: - CMutexLock(Mutex& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : lock(mutexIn, boost::defer_lock) + CCriticalBlock(CCriticalSection& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : lock(mutexIn, std::defer_lock) { if (fTry) TryEnter(pszName, pszFile, nLine); @@ -144,18 +149,18 @@ public: Enter(pszName, pszFile, nLine); } - CMutexLock(Mutex* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) + CCriticalBlock(CCriticalSection* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) return; - lock = boost::unique_lock<Mutex>(*pmutexIn, boost::defer_lock); + lock = std::unique_lock<CCriticalSection>(*pmutexIn, std::defer_lock); if (fTry) TryEnter(pszName, pszFile, nLine); else Enter(pszName, pszFile, nLine); } - ~CMutexLock() UNLOCK_FUNCTION() + ~CCriticalBlock() UNLOCK_FUNCTION() { if (lock.owns_lock()) LeaveCritical(); @@ -167,8 +172,6 @@ public: } }; -typedef CMutexLock<CCriticalSection> CCriticalBlock; - #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) @@ -191,8 +194,8 @@ typedef CMutexLock<CCriticalSection> CCriticalBlock; class CSemaphore { private: - boost::condition_variable condition; - boost::mutex mutex; + std::condition_variable condition; + std::mutex mutex; int value; public: @@ -200,16 +203,14 @@ public: void wait() { - boost::unique_lock<boost::mutex> lock(mutex); - while (value < 1) { - condition.wait(lock); - } + std::unique_lock<std::mutex> lock(mutex); + condition.wait(lock, [&]() { return value >= 1; }); value--; } bool try_wait() { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); if (value < 1) return false; value--; @@ -219,7 +220,7 @@ public: void post() { { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); value++; } condition.notify_one(); diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index b88ad5ed1b..6be176a1ad 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -4,17 +4,17 @@ // Unit tests for denial-of-service detection/prevention code -#include "chainparams.h" -#include "keystore.h" -#include "net.h" -#include "net_processing.h" -#include "pow.h" -#include "script/sign.h" -#include "serialize.h" -#include "util.h" -#include "validation.h" - -#include "test/test_bitcoin.h" +#include <chainparams.h> +#include <keystore.h> +#include <net.h> +#include <net_processing.h> +#include <pow.h> +#include <script/sign.h> +#include <serialize.h> +#include <util.h> +#include <validation.h> + +#include <test/test_bitcoin.h> #include <stdint.h> @@ -40,8 +40,141 @@ CService ip(uint32_t i) static NodeId id = 0; +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); + BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) +// Test eviction of an outbound peer whose chain never advances +// Mock a node connection, and use mocktime to simulate a peer +// which never sends any headers messages. PeerLogic should +// decide to evict that outbound peer, after the appropriate timeouts. +// Note that we protect 4 outbound nodes from being subject to +// this logic; this test takes advantage of that protection only +// being applied to nodes which send headers with sufficient +// work. +BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) +{ + std::atomic<bool> interruptDummy(false); + + // Mock an outbound peer + CAddress addr1(ip(0xa0b0c001), NODE_NONE); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + + peerLogic->InitializeNode(&dummyNode1); + dummyNode1.nVersion = 1; + dummyNode1.fSuccessfullyConnected = true; + + // This test requires that we have a chain with non-zero work. + LOCK(cs_main); + BOOST_CHECK(chainActive.Tip() != nullptr); + BOOST_CHECK(chainActive.Tip()->nChainWork > 0); + + // Test starts here + LOCK(dummyNode1.cs_sendProcessing); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + LOCK(dummyNode1.cs_vSend); + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + dummyNode1.vSendMsg.clear(); + + int64_t nStartTime = GetTime(); + // Wait 21 minutes + SetMockTime(nStartTime+21*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + // Wait 3 more minutes + SetMockTime(nStartTime+24*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in disconnect + BOOST_CHECK(dummyNode1.fDisconnect == true); + SetMockTime(0); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); +} + +void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidation &peerLogic) +{ + CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + CNode &node = *vNodes.back(); + node.SetSendVersion(PROTOCOL_VERSION); + + peerLogic.InitializeNode(&node); + node.nVersion = 1; + node.fSuccessfullyConnected = true; + + CConnmanTest::AddNode(node); +} + +BOOST_AUTO_TEST_CASE(stale_tip_peer_management) +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + constexpr int nMaxOutbound = 8; + CConnman::Options options; + options.nMaxConnections = 125; + options.nMaxOutbound = nMaxOutbound; + options.nMaxFeeler = 1; + + connman->Init(options); + std::vector<CNode *> vNodes; + + // Mock some outbound peers + for (int i=0; i<nMaxOutbound; ++i) { + AddRandomOutboundPeer(vNodes, *peerLogic); + } + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + + // No nodes should be marked for disconnection while we have no extra peers + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1); + + // Now tip should definitely be stale, and we should look for an extra + // outbound peer + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + BOOST_CHECK(connman->GetTryNewOutboundPeer()); + + // Still no peers should be marked for disconnection + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + // If we add one more peer, something should get marked for eviction + // on the next check (since we're mocking the time to be in the future, the + // required time connected check should be satisfied). + AddRandomOutboundPeer(vNodes, *peerLogic); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; i<nMaxOutbound; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + // Last added node should get marked for eviction + BOOST_CHECK(vNodes.back()->fDisconnect == true); + + vNodes.back()->fDisconnect = false; + + // Update the last announced block time for the last + // peer, and check that the next newest node gets evicted. + UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; i<nMaxOutbound-1; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes[nMaxOutbound-1]->fDisconnect == true); + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + bool dummy; + for (const CNode *node : vNodes) { + peerLogic->FinalizeNode(node->GetId(), dummy); + } + + CConnmanTest::ClearNodes(); +} + BOOST_AUTO_TEST_CASE(DoS_banning) { std::atomic<bool> interruptDummy(false); @@ -53,7 +186,11 @@ BOOST_AUTO_TEST_CASE(DoS_banning) peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; - Misbehaving(dummyNode1.GetId(), 100); // Should get banned + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 100); // Should get banned + } + LOCK(dummyNode1.cs_sendProcessing); peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned @@ -64,13 +201,24 @@ BOOST_AUTO_TEST_CASE(DoS_banning) peerLogic->InitializeNode(&dummyNode2); dummyNode2.nVersion = 1; dummyNode2.fSuccessfullyConnected = true; - Misbehaving(dummyNode2.GetId(), 50); + { + LOCK(cs_main); + Misbehaving(dummyNode2.GetId(), 50); + } + LOCK(dummyNode2.cs_sendProcessing); peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr2)); // 2 not banned yet... BOOST_CHECK(connman->IsBanned(addr1)); // ... but 1 still should be - Misbehaving(dummyNode2.GetId(), 50); + { + LOCK(cs_main); + Misbehaving(dummyNode2.GetId(), 50); + } peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(connman->IsBanned(addr2)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); + peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_banscore) @@ -85,16 +233,29 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; - Misbehaving(dummyNode1.GetId(), 100); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 100); + } + LOCK(dummyNode1.cs_sendProcessing); peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); - Misbehaving(dummyNode1.GetId(), 10); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 10); + } peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); - Misbehaving(dummyNode1.GetId(), 1); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 1); + } peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); gArgs.ForceSetArg("-banscore", std::to_string(DEFAULT_BANSCORE_THRESHOLD)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_bantime) @@ -112,7 +273,11 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) dummyNode.nVersion = 1; dummyNode.fSuccessfullyConnected = true; - Misbehaving(dummyNode.GetId(), 100); + { + LOCK(cs_main); + Misbehaving(dummyNode.GetId(), 100); + } + LOCK(dummyNode.cs_sendProcessing); peerLogic->SendMessages(&dummyNode, interruptDummy); BOOST_CHECK(connman->IsBanned(addr)); @@ -121,11 +286,15 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime+60*60*24+1); BOOST_CHECK(!connman->IsBanned(addr)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode.GetId(), dummy); } CTransactionRef RandomOrphan() { std::map<uint256, COrphanTx>::iterator it; + LOCK(cs_main); it = mapOrphanTransactions.lower_bound(InsecureRand256()); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); @@ -195,6 +364,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(!AddOrphanTx(MakeTransactionRef(tx), i)); } + LOCK(cs_main); // Test EraseOrphansFor: for (NodeId i = 0; i < 3; i++) { diff --git a/src/test/README.md b/src/test/README.md index dbaa9c27f3..01da32109b 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -52,4 +52,4 @@ examine `uint256_tests.cpp`. For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: -[http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). +[http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://archive.is/dRBGf). diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 7be29c6d6b..609b1a05d3 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" -#include "test/test_bitcoin.h" +#include <addrman.h> +#include <test/test_bitcoin.h> #include <string> #include <boost/test/unit_test.hpp> -#include "hash.h" -#include "netbase.h" -#include "random.h" +#include <hash.h> +#include <netbase.h> +#include <random.h> class CAddrManTest : public CAddrMan { diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 4a533b5bf2..83fa3b76be 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" +#include <util.h> -#include "support/allocators/secure.h" -#include "test/test_bitcoin.h" +#include <support/allocators/secure.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 952cf901f0..7ebbd22c5d 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" -#include "policy/feerate.h" -#include "test/test_bitcoin.h" +#include <amount.h> +#include <policy/feerate.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 2c98fbcfd6..eb3148a1de 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -8,11 +8,11 @@ #include <iomanip> #include <limits> #include <cmath> -#include "uint256.h" -#include "arith_uint256.h" +#include <uint256.h> +#include <arith_uint256.h> #include <string> -#include "version.h" -#include "test/test_bitcoin.h" +#include <version.h> +#include <test/test_bitcoin.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 6422b3a88f..abf236b953 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 6bc6dd5187..3156cc53d0 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" - -#include "data/base58_encode_decode.json.h" -#include "data/base58_keys_invalid.json.h" -#include "data/base58_keys_valid.json.h" - -#include "key.h" -#include "script/script.h" -#include "test/test_bitcoin.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" +#include <base58.h> + +#include <test/data/base58_encode_decode.json.h> +#include <test/data/base58_keys_invalid.json.h> +#include <test/data/base58_keys_valid.json.h> + +#include <key.h> +#include <script/script.h> +#include <test/test_bitcoin.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> #include <univalue.h> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index ccad94d946..46b3a04f35 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index ce4cddd64b..495290c8d9 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bech32.h" -#include "test/test_bitcoin.h" +#include <bech32.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index e123c26ad0..e3e727439b 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -4,12 +4,12 @@ #include <boost/test/unit_test.hpp> -#include "base58.h" -#include "key.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <base58.h> +#include <key.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index f2d5b385d0..718b5bdfb9 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "blockencodings.h" -#include "consensus/merkle.h" -#include "chainparams.h" -#include "random.h" +#include <blockencodings.h> +#include <consensus/merkle.h> +#include <chainparams.h> +#include <random.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> @@ -62,6 +62,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT @@ -161,6 +162,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -227,6 +229,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[1]->GetHash(), entry.FromTx(*block.vtx[1])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index eac2c102a6..fdd30be0b1 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -2,20 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bloom.h" - -#include "base58.h" -#include "clientversion.h" -#include "key.h" -#include "merkleblock.h" -#include "primitives/block.h" -#include "random.h" -#include "serialize.h" -#include "streams.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <bloom.h> + +#include <base58.h> +#include <clientversion.h> +#include <key.h> +#include <merkleblock.h> +#include <primitives/block.h> +#include <random.h> +#include <serialize.h> +#include <streams.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 7b3134d327..ab4f08b999 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compat/byteswap.h" -#include "test/test_bitcoin.h" +#include <compat/byteswap.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 6ae0bcadd0..42f9dd0600 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" -#include "utiltime.h" -#include "validation.h" +#include <util.h> +#include <utiltime.h> +#include <validation.h> -#include "test/test_bitcoin.h" -#include "checkqueue.h" +#include <test/test_bitcoin.h> +#include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> #include <atomic> @@ -18,13 +18,13 @@ #include <unordered_set> #include <memory> -#include "random.h" +#include <random.h> // BasicTestingSetup not sufficient because nScriptCheckThreads is not set // otherwise. BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) -static const int QUEUE_BATCH_SIZE = 128; +static const unsigned int QUEUE_BATCH_SIZE = 128; struct FakeCheck { bool operator()() @@ -38,7 +38,7 @@ struct FakeCheckCheckCompletion { static std::atomic<size_t> n_calls; bool operator()() { - ++n_calls; + n_calls.fetch_add(1, std::memory_order_relaxed); return true; } void swap(FakeCheckCheckCompletion& x){}; @@ -88,15 +88,15 @@ struct MemoryCheck { // // Really, copy constructor should be deletable, but CCheckQueue breaks // if it is deleted because of internal push_back. - fake_allocated_memory += b; + fake_allocated_memory.fetch_add(b, std::memory_order_relaxed); }; MemoryCheck(bool b_) : b(b_) { - fake_allocated_memory += b; + fake_allocated_memory.fetch_add(b, std::memory_order_relaxed); }; - ~MemoryCheck(){ - fake_allocated_memory -= b; - + ~MemoryCheck() + { + fake_allocated_memory.fetch_sub(b, std::memory_order_relaxed); }; void swap(MemoryCheck& x) { std::swap(b, x.b); }; }; @@ -117,9 +117,9 @@ struct FrozenCleanupCheck { { if (should_freeze) { std::unique_lock<std::mutex> l(m); - nFrozen = 1; + nFrozen.store(1, std::memory_order_relaxed); cv.notify_one(); - cv.wait(l, []{ return nFrozen == 0;}); + cv.wait(l, []{ return nFrozen.load(std::memory_order_relaxed) == 0;}); } } void swap(FrozenCleanupCheck& x){std::swap(should_freeze, x.should_freeze);}; @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) control.Add(vChecks); } bool r =control.Wait(); - BOOST_REQUIRE(r || end_fails); + BOOST_REQUIRE(r != end_fails); } } tg.interrupt_all(); @@ -337,7 +337,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) tg.join_all(); } -// Test that a new verification cannot occur until all checks +// Test that a new verification cannot occur until all checks // have been destructed BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { @@ -361,11 +361,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); // Wait until the queue has finished all jobs and frozen FrozenCleanupCheck::cv.wait(l, [](){return FrozenCleanupCheck::nFrozen == 1;}); - // Try to get control of the queue a bunch of times - for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); - } - // Unfreeze + } + // Try to get control of the queue a bunch of times + for (auto x = 0; x < 100 && !fails; ++x) { + fails = queue->ControlMutex.try_lock(); + } + { + // Unfreeze (we need lock n case of spurious wakeup) + std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); FrozenCleanupCheck::nFrozen = 0; } // Awaken frozen destructor diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index dc358bff95..ed556c07f4 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coins.h" -#include "script/standard.h" -#include "uint256.h" -#include "undo.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "consensus/validation.h" +#include <coins.h> +#include <script/standard.h> +#include <uint256.h> +#include <undo.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <consensus/validation.h> #include <vector> #include <map> @@ -81,8 +81,8 @@ public: // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); size_t count = 0; - for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coin.DynamicMemoryUsage(); + for (const auto& entry : cacheCoins) { + ret += entry.second.coin.DynamicMemoryUsage(); ++count; } BOOST_CHECK_EQUAL(GetCacheSize(), count); @@ -189,15 +189,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Once every 1000 iterations and at the end, verify the full cache. if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (auto it = result.begin(); it != result.end(); it++) { - bool have = stack.back()->HaveCoin(it->first); - const Coin& coin = stack.back()->AccessCoin(it->first); + for (const auto& entry : result) { + bool have = stack.back()->HaveCoin(entry.first); + const Coin& coin = stack.back()->AccessCoin(entry.first); BOOST_CHECK(have == !coin.IsSpent()); - BOOST_CHECK(coin == it->second); + BOOST_CHECK(coin == entry.second); if (coin.IsSpent()) { missed_an_entry = true; } else { - BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + BOOST_CHECK(stack.back()->HaveCoinInCache(entry.first)); found_an_entry = true; } } @@ -420,11 +420,11 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // Once every 1000 iterations and at the end, verify the full cache. if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (auto it = result.begin(); it != result.end(); it++) { - bool have = stack.back()->HaveCoin(it->first); - const Coin& coin = stack.back()->AccessCoin(it->first); + for (const auto& entry : result) { + bool have = stack.back()->HaveCoin(entry.first); + const Coin& coin = stack.back()->AccessCoin(entry.first); BOOST_CHECK(have == !coin.IsSpent()); - BOOST_CHECK(coin == it->second); + BOOST_CHECK(coin == entry.second); } } diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 35e4458bba..87e75d8020 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compressor.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <compressor.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index c748b2448c..125b68e76d 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -2,17 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/aes.h" -#include "crypto/chacha20.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "crypto/sha512.h" -#include "crypto/hmac_sha256.h" -#include "crypto/hmac_sha512.h" -#include "random.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <crypto/aes.h> +#include <crypto/chacha20.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> +#include <crypto/hmac_sha256.h> +#include <crypto/hmac_sha512.h> +#include <random.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> @@ -340,6 +340,22 @@ BOOST_AUTO_TEST_CASE(hmac_sha256_testvectors) { "647320746f20626520686173686564206265666f7265206265696e6720757365" "642062792074686520484d414320616c676f726974686d2e", "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2"); + // Test case with key length 63 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a6566", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "9de4b546756c83516720a4ad7fe7bdbeac4298c6fdd82b15f895a6d10b0769a6"); + // Test case with key length 64 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "528c609a4c9254c274585334946b7c2661bad8f1fc406b20f6892478d19163dd"); + // Test case with key length 65 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "d06af337f359a2330deffb8e3cbe4b5b7aa8ca1f208528cdbd245d5dc63c4483"); } BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { @@ -383,6 +399,31 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { "642062792074686520484d414320616c676f726974686d2e", "e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944" "b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58"); + // Test case with key length 127 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a6566", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "267424dfb8eeb999f3e5ec39a4fe9fd14c923e6187e0897063e5c9e02b2e624a" + "c04413e762977df71a9fb5d562b37f89dfdfb930fce2ed1fa783bbc2a203d80e"); + // Test case with key length 128 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "43aaac07bb1dd97c82c04df921f83b16a68d76815cd1a30d3455ad43a3d80484" + "2bb35462be42cc2e4b5902de4d204c1c66d93b47d1383e3e13a3788687d61258"); + // Test case with key length 129 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "0b273325191cfc1b4b71d5075c8fcad67696309d292b1dad2cd23983a35feb8e" + "fb29795e79f2ef27f68cb1e16d76178c307a67beaad9456fac5fdffeadb16e2c"); } BOOST_AUTO_TEST_CASE(aes_testvectors) { diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 1004482224..32d46da984 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> -#include "cuckoocache.h" -#include "script/sigcache.h" -#include "test/test_bitcoin.h" -#include "random.h" +#include <cuckoocache.h> +#include <script/sigcache.h> +#include <test/test_bitcoin.h> +#include <random.h> #include <thread> /** Test Suite for CuckooCache diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 698e898231..63f43c0fc6 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -862,8 +862,6 @@ ["Ensure 100% coverage of discouraged NOPS"], ["1", "NOP1", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], -["1", "CHECKLOCKTIMEVERIFY", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], -["1", "CHECKSEQUENCEVERIFY", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP4", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP5", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP6", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index efddafe17e..2e1e4da72f 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "dbwrapper.h" -#include "uint256.h" -#include "random.h" -#include "test/test_bitcoin.h" +#include <dbwrapper.h> +#include <uint256.h> +#include <random.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + std::unique_ptr<CDBWrapper> dbw = MakeUnique<CDBWrapper>(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = InsecureRand256(); uint256 res; @@ -135,8 +135,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); // Call the destructor to free leveldb LOCK - delete dbw; - dbw = nullptr; + dbw.reset(); // Now, set up another wrapper that wants to obfuscate the same directory CDBWrapper odbw(ph, (1 << 10), false, false, true); @@ -167,7 +166,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + std::unique_ptr<CDBWrapper> dbw = MakeUnique<CDBWrapper>(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = InsecureRand256(); uint256 res; @@ -177,8 +176,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); // Call the destructor to free leveldb LOCK - delete dbw; - dbw = nullptr; + dbw.reset(); // Simulate a -reindex by wiping the existing data store CDBWrapper odbw(ph, (1 << 10), false, true, true); diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 40f0ecd5f1..3c39258693 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" -#include "test/test_bitcoin.h" +#include <util.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 05b6b3b1e6..68f445d142 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "hash.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <hash.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 91c0175412..7b9d7a6398 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" - -#include "base58.h" -#include "script/script.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <key.h> + +#include <base58.h> +#include <script/script.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index b071ab117b..90730cd7b5 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "limitedmap.h" +#include <limitedmap.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index 656aec606b..4be32f7628 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "validation.h" -#include "net.h" +#include <chainparams.h> +#include <validation.h> +#include <net.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 116210a297..a694b870fe 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/policy.h" -#include "txmempool.h" -#include "util.h" +#include <policy/policy.h> +#include <txmempool.h> +#include <util.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <list> @@ -165,6 +165,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) sortedOrder[2] = tx1.GetHash().ToString(); // 10000 sortedOrder[3] = tx4.GetHash().ToString(); // 15000 sortedOrder[4] = tx2.GetHash().ToString(); // 20000 + LOCK(pool.cs); CheckSort<descendant_score>(pool, sortedOrder); /* low fee but with high fee child */ @@ -375,6 +376,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) } sortedOrder[4] = tx3.GetHash().ToString(); // 0 + LOCK(pool.cs); CheckSort<ancestor_score>(pool, sortedOrder); /* low fee parent with high fee child */ diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 1a1cf4399c..ee48d96ad9 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/merkle.h" -#include "test/test_bitcoin.h" +#include <consensus/merkle.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 3e66c6f2c6..37a1a84136 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkleblock.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <merkleblock.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 41e0626eb9..97c548dbb0 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -2,23 +2,23 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "consensus/merkle.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "validation.h" -#include "miner.h" -#include "policy/policy.h" -#include "pubkey.h" -#include "script/standard.h" -#include "txmempool.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" - -#include "test/test_bitcoin.h" +#include <chainparams.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <validation.h> +#include <miner.h> +#include <policy/policy.h> +#include <pubkey.h> +#include <script/standard.h> +#include <txmempool.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> + +#include <test/test_bitcoin.h> #include <memory> @@ -26,6 +26,17 @@ BOOST_FIXTURE_TEST_SUITE(miner_tests, TestingSetup) +// BOOST_CHECK_EXCEPTION predicates to check the specific validation error +class HasReason { +public: + HasReason(const std::string& reason) : m_reason(reason) {} + bool operator() (const std::runtime_error& e) const { + return std::string(e.what()).find(m_reason) != std::string::npos; + }; +private: + const std::string m_reason; +}; + static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); static BlockAssembler AssemblerForTest(const CChainParams& params) { @@ -264,7 +275,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); mempool.clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -304,7 +316,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // orphan in mempool, template creation fails hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); mempool.clear(); // child with higher feerate than parent @@ -332,24 +344,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); // give it a fee so it'll get mined mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); - mempool.clear(); - - // invalid (pre-p2sh) txn in mempool, template creation fails - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vin[0].prevout.n = 0; - tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; - script = CScript() << OP_0; - tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script)); - hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - tx.vin[0].prevout.hash = hash; - tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); - tx.vout[0].nValue -= LOWFEE; - hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + // Should throw bad-cb-multiple + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); mempool.clear(); // double spend txn pair in mempool, template creation fails @@ -362,7 +358,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); mempool.clear(); // subsidy changing @@ -391,6 +387,25 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) chainActive.SetTip(next); } BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + + // invalid p2sh txn in mempool, template creation fails + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; + script = CScript() << OP_0; + tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script)); + hash = tx.GetHash(); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + // Should throw block-validation-failed + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); + mempool.clear(); + // Delete the dummy blocks again. while (chainActive.Tip()->nHeight > nHeight) { CBlockIndex* del = chainActive.Tip(); diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index de7f3b48f5..ffba7d6d22 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -2,16 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" -#include "keystore.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/interpreter.h" -#include "script/sign.h" -#include "script/ismine.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <key.h> +#include <keystore.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/interpreter.h> +#include <script/sign.h> +#include <script/ismine.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 31b05d868b..b4176a2b85 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,17 +1,17 @@ // Copyright (c) 2012-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" -#include "test/test_bitcoin.h" +#include <addrman.h> +#include <test/test_bitcoin.h> #include <string> #include <boost/test/unit_test.hpp> -#include "hash.h" -#include "serialize.h" -#include "streams.h" -#include "net.h" -#include "netbase.h" -#include "chainparams.h" -#include "util.h" +#include <hash.h> +#include <serialize.h> +#include <streams.h> +#include <net.h> +#include <netbase.h> +#include <chainparams.h> +#include <util.h> class CAddrManSerializationMock : public CAddrMan { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 1baf7643e5..f3a9b1fc65 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "netbase.h" -#include "test/test_bitcoin.h" -#include "utilstrencodings.h" +#include <netbase.h> +#include <test/test_bitcoin.h> +#include <utilstrencodings.h> #include <string> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index c1d216d094..b7b632be3a 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/merkle.h" -#include "merkleblock.h" -#include "serialize.h" -#include "streams.h" -#include "uint256.h" -#include "arith_uint256.h" -#include "version.h" -#include "test/test_bitcoin.h" +#include <consensus/merkle.h> +#include <merkleblock.h> +#include <serialize.h> +#include <streams.h> +#include <uint256.h> +#include <arith_uint256.h> +#include <version.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index fd8f7191f4..d8026e4468 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/policy.h" -#include "policy/fees.h" -#include "txmempool.h" -#include "uint256.h" -#include "util.h" +#include <policy/policy.h> +#include <policy/fees.h> +#include <txmempool.h> +#include <uint256.h> +#include <util.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index b13f2625aa..2837537c9a 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "pow.h" -#include "random.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <chain.h> +#include <chainparams.h> +#include <pow.h> +#include <random.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 841282873f..1327d67650 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -3,13 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <vector> -#include "prevector.h" +#include <prevector.h> -#include "reverse_iterator.h" -#include "serialize.h" -#include "streams.h" +#include <reverse_iterator.h> +#include <serialize.h> +#include <streams.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 0d541ec7d4..781b1e242d 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -10,9 +10,9 @@ #include <map> #include <stdlib.h> -#include "support/events.h" +#include <support/events.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 132e190051..1ca5a53d72 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" +#include <random.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 00dc47e13e..86d6161e20 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "reverselock.h" -#include "test/test_bitcoin.h" +#include <reverselock.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index c6643be7a7..c69b5f1ca3 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" -#include "rpc/client.h" +#include <rpc/server.h> +#include <rpc/client.h> -#include "base58.h" -#include "core_io.h" -#include "netbase.h" +#include <base58.h> +#include <core_io.h> +#include <netbase.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> @@ -65,7 +65,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_EQUAL(find_value(r.get_obj(), "size").get_int(), 193); BOOST_CHECK_EQUAL(find_value(r.get_obj(), "version").get_int(), 1); BOOST_CHECK_EQUAL(find_value(r.get_obj(), "locktime").get_int(), 0); - BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); + BOOST_CHECK_THROW(CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); + BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false")); + BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false extra"), std::runtime_error); BOOST_CHECK_THROW(CallRPC("signrawtransaction"), std::runtime_error); BOOST_CHECK_THROW(CallRPC("signrawtransaction null"), std::runtime_error); diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 51f9e9f39f..b3ec6d34c7 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compat/sanity.h" -#include "key.h" -#include "test/test_bitcoin.h" +#include <compat/sanity.h> +#include <key.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 1de865776e..7e86eed1c5 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" -#include "scheduler.h" +#include <random.h> +#include <scheduler.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/bind.hpp> #include <boost/thread.hpp> diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 58aa32c969..5a64aa3959 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -2,17 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "script/ismine.h" -#include "test/test_bitcoin.h" +#include <consensus/tx_verify.h> +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <script/ismine.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index bd2d9ed115..19060eccc9 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" -#include "keystore.h" -#include "script/ismine.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/standard.h" -#include "test/test_bitcoin.h" +#include <key.h> +#include <keystore.h> +#include <script/ismine.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/standard.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 011a5db795..f96d867bc6 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -2,21 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/script_tests.json.h" - -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" -#include "rpc/server.h" +#include <test/data/script_tests.json.h> + +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> +#include <rpc/server.h> #if defined(HAVE_CONSENSUS_LIB) -#include "script/bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> #endif #include <fstream> @@ -166,6 +166,17 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message); BOOST_CHECK_MESSAGE(err == scriptError, std::string(FormatScriptError(err)) + " where " + std::string(FormatScriptError((ScriptError_t)scriptError)) + " expected: " + message); + + // Verify that removing flags from a passing test or adding flags to a failing test does not change the result. + for (int i = 0; i < 16; ++i) { + int extra_flags = InsecureRandBits(16); + int combined_flags = expect ? (flags & ~extra_flags) : (flags | extra_flags); + // Weed out some invalid flag combinations. + if (combined_flags & SCRIPT_VERIFY_CLEANSTACK && ~combined_flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) continue; + if (combined_flags & SCRIPT_VERIFY_WITNESS && ~combined_flags & SCRIPT_VERIFY_P2SH) continue; + BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, combined_flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message + strprintf(" (with flags %x)", combined_flags)); + } + #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << tx2; diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 94dd58526c..ad15427a9b 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -12,7 +12,7 @@ #include <stdint.h> #include <string> #include <vector> -#include "assert.h" +#include <assert.h> class scriptnum10_error : public std::runtime_error { diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 280eb59ce8..aef500174a 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "scriptnum10.h" -#include "script/script.h" -#include "test/test_bitcoin.h" +#include <test/scriptnum10.h> +#include <script/script.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <limits.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 9661a66514..28ed2bfeb0 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "serialize.h" -#include "streams.h" -#include "hash.h" -#include "test/test_bitcoin.h" +#include <serialize.h> +#include <streams.h> +#include <hash.h> +#include <test/test_bitcoin.h> #include <stdint.h> diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index ecbdf57788..d97e106287 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "data/sighash.json.h" -#include "hash.h" -#include "script/interpreter.h" -#include "script/script.h" -#include "serialize.h" -#include "streams.h" -#include "test/test_bitcoin.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <test/data/sighash.json.h> +#include <hash.h> +#include <script/interpreter.h> +#include <script/script.h> +#include <serialize.h> +#include <streams.h> +#include <test/test_bitcoin.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <iostream> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index d3b8b07228..891cd1e1d8 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "pubkey.h" -#include "key.h" -#include "script/script.h" -#include "script/standard.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <pubkey.h> +#include <key.h> +#include <script/script.h> +#include <script/standard.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 164cbd873f..1ffd9f2b42 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <chain.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index af2a152aa5..3636e6c332 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "streams.h" -#include "support/allocators/zeroafterfree.h" -#include "test/test_bitcoin.h" +#include <streams.h> +#include <support/allocators/zeroafterfree.h> +#include <test/test_bitcoin.h> #include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <boost/test/unit_test.hpp> diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 79bc48a118..f52c8ccc21 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -2,29 +2,35 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "test_bitcoin.h" - -#include "chainparams.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "crypto/sha256.h" -#include "fs.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "net_processing.h" -#include "pubkey.h" -#include "random.h" -#include "txdb.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "streams.h" -#include "rpc/server.h" -#include "rpc/register.h" -#include "script/sigcache.h" +#include <test/test_bitcoin.h> + +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <crypto/sha256.h> +#include <validation.h> +#include <miner.h> +#include <net_processing.h> +#include <ui_interface.h> +#include <streams.h> +#include <rpc/server.h> +#include <rpc/register.h> +#include <script/sigcache.h> #include <memory> +void CConnmanTest::AddNode(CNode& node) +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.push_back(&node); +} + +void CConnmanTest::ClearNodes() +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.clear(); +} + uint256 insecure_rand_seed = GetRandHash(); FastRandomContext insecure_rand_ctx(insecure_rand_seed); @@ -69,9 +75,9 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); mempool.setSanityCheck(1.0); - pblocktree = new CBlockTreeDB(1 << 20, true); - pcoinsdbview = new CCoinsViewDB(1 << 23, true); - pcoinsTip = new CCoinsViewCache(pcoinsdbview); + pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); if (!LoadGenesisBlock(chainparams)) { throw std::runtime_error("LoadGenesisBlock failed."); } @@ -86,7 +92,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha threadGroup.create_thread(&ThreadScriptCheck); g_connman = std::unique_ptr<CConnman>(new CConnman(0x1337, 0x1337)); // Deterministic randomness for tests. connman = g_connman.get(); - peerLogic.reset(new PeerLogicValidation(connman)); + peerLogic.reset(new PeerLogicValidation(connman, scheduler)); } TestingSetup::~TestingSetup() @@ -98,14 +104,17 @@ TestingSetup::~TestingSetup() g_connman.reset(); peerLogic.reset(); UnloadBlockIndex(); - delete pcoinsTip; - delete pcoinsdbview; - delete pblocktree; + pcoinsTip.reset(); + pcoinsdbview.reset(); + pblocktree.reset(); fs::remove_all(pathTemp); } TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) { + // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. + // TODO: fix the code to support SegWit blocks. + UpdateVersionBitsParameters(Consensus::DEPLOYMENT_SEGWIT, 0, Consensus::BIP9Deployment::NO_TIMEOUT); // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; @@ -134,7 +143,10 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& block.vtx.push_back(MakeTransactionRef(tx)); // IncrementExtraNonce creates a valid coinbase and merkleRoot unsigned int extraNonce = 0; - IncrementExtraNonce(&block, chainActive.Tip(), extraNonce); + { + LOCK(cs_main); + IncrementExtraNonce(&block, chainActive.Tip(), extraNonce); + } while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 2390aca342..80a9cbaeea 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -5,14 +5,14 @@ #ifndef BITCOIN_TEST_TEST_BITCOIN_H #define BITCOIN_TEST_TEST_BITCOIN_H -#include "chainparamsbase.h" -#include "fs.h" -#include "key.h" -#include "pubkey.h" -#include "random.h" -#include "scheduler.h" -#include "txdb.h" -#include "txmempool.h" +#include <chainparamsbase.h> +#include <fs.h> +#include <key.h> +#include <pubkey.h> +#include <random.h> +#include <scheduler.h> +#include <txdb.h> +#include <txmempool.h> #include <boost/thread.hpp> @@ -49,9 +49,14 @@ struct BasicTestingSetup { * Included are data directory, coins database, script check threads setup. */ class CConnman; +class CNode; +struct CConnmanTest { + static void AddNode(CNode& node); + static void ClearNodes(); +}; + class PeerLogicValidation; struct TestingSetup: public BasicTestingSetup { - CCoinsViewDB *pcoinsdbview; fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index 581ad2ffa0..319d81187a 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -3,22 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "consensus/merkle.h" -#include "primitives/block.h" -#include "script/script.h" -#include "addrman.h" -#include "chain.h" -#include "coins.h" -#include "compressor.h" -#include "net.h" -#include "protocol.h" -#include "streams.h" -#include "undo.h" -#include "version.h" -#include "pubkey.h" +#include <consensus/merkle.h> +#include <primitives/block.h> +#include <script/script.h> +#include <addrman.h> +#include <chain.h> +#include <coins.h> +#include <compressor.h> +#include <net.h> +#include <protocol.h> +#include <streams.h> +#include <undo.h> +#include <version.h> +#include <pubkey.h> +#include <blockencodings.h> #include <stdint.h> #include <unistd.h> @@ -45,6 +46,8 @@ enum TEST_ID { CBLOOMFILTER_DESERIALIZE, CDISKBLOCKINDEX_DESERIALIZE, CTXOUTCOMPRESSOR_DESERIALIZE, + BLOCKTRANSACTIONS_DESERIALIZE, + BLOCKTRANSACTIONSREQUEST_DESERIALIZE, TEST_ID_END }; @@ -245,6 +248,26 @@ int test_one_input(std::vector<uint8_t> buffer) { break; } + case BLOCKTRANSACTIONS_DESERIALIZE: + { + try + { + BlockTransactions bt; + ds >> bt; + } catch (const std::ios_base::failure& e) {return 0;} + + break; + } + case BLOCKTRANSACTIONSREQUEST_DESERIALIZE: + { + try + { + BlockTransactionsRequest btr; + ds >> btr; + } catch (const std::ios_base::failure& e) {return 0;} + + break; + } default: return 0; } diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp index b556c953b9..8d81ff6fc6 100644 --- a/src/test/test_bitcoin_main.cpp +++ b/src/test/test_bitcoin_main.cpp @@ -4,7 +4,7 @@ #define BOOST_TEST_MODULE Bitcoin Test Suite -#include "net.h" +#include <net.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 34863fd9d0..9ff95bd2b0 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include "timedata.h" -#include "test/test_bitcoin.h" +#include <timedata.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp index b7affaacde..d0aa8659c2 100644 --- a/src/test/torcontrol_tests.cpp +++ b/src/test/torcontrol_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include "test/test_bitcoin.h" -#include "torcontrol.cpp" +#include <test/test_bitcoin.h> +#include <torcontrol.cpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index cb6ab7cdbe..34a182722d 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -2,24 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/tx_invalid.json.h" -#include "data/tx_valid.json.h" -#include "test/test_bitcoin.h" - -#include "clientversion.h" -#include "checkqueue.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/sign.h" -#include "script/script_error.h" -#include "script/standard.h" -#include "utilstrencodings.h" +#include <test/data/tx_invalid.json.h> +#include <test/data/tx_valid.json.h> +#include <test/test_bitcoin.h> + +#include <clientversion.h> +#include <checkqueue.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/sign.h> +#include <script/script_error.h> +#include <script/standard.h> +#include <utilstrencodings.h> #include <map> #include <string> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp new file mode 100644 index 0000000000..2d1eb7b772 --- /dev/null +++ b/src/test/txvalidation_tests.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <validation.h> +#include <txmempool.h> +#include <amount.h> +#include <consensus/validation.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + + +BOOST_AUTO_TEST_SUITE(txvalidation_tests) + +/** + * Ensure that the mempool won't accept coinbase transactions. + */ +BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) +{ + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CMutableTransaction coinbaseTx; + + coinbaseTx.nVersion = 1; + coinbaseTx.vin.resize(1); + coinbaseTx.vout.resize(1); + coinbaseTx.vin[0].scriptSig = CScript() << OP_11 << OP_EQUAL; + coinbaseTx.vout[0].nValue = 1 * CENT; + coinbaseTx.vout[0].scriptPubKey = scriptPubKey; + + assert(CTransaction(coinbaseTx).IsCoinBase()); + + CValidationState state; + + LOCK(cs_main); + + unsigned int initialPoolSize = mempool.size(); + + BOOST_CHECK_EQUAL( + false, + AcceptToMemoryPool(mempool, state, MakeTransactionRef(coinbaseTx), + nullptr /* pfMissingInputs */, + nullptr /* plTxnReplaced */, + true /* bypass_limits */, + 0 /* nAbsurdFee */)); + + // Check that the transaction hasn't been added to mempool. + BOOST_CHECK_EQUAL(mempool.size(), initialPoolSize); + + // Check that the validation state reflects the unsuccessful attempt. + BOOST_CHECK(state.IsInvalid()); + BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); + + int nDoS; + BOOST_CHECK_EQUAL(state.IsInvalid(nDoS), true); + BOOST_CHECK_EQUAL(nDoS, 100); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 82ca93e7da..fe8cb6126e 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -2,20 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/validation.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "pubkey.h" -#include "txmempool.h" -#include "random.h" -#include "script/standard.h" -#include "script/sign.h" -#include "test/test_bitcoin.h" -#include "utiltime.h" -#include "core_io.h" -#include "keystore.h" -#include "policy/policy.h" +#include <consensus/validation.h> +#include <key.h> +#include <validation.h> +#include <miner.h> +#include <pubkey.h> +#include <txmempool.h> +#include <random.h> +#include <script/standard.h> +#include <script/sign.h> +#include <test/test_bitcoin.h> +#include <utiltime.h> +#include <core_io.h> +#include <keystore.h> +#include <policy/policy.h> #include <boost/test/unit_test.hpp> @@ -66,6 +66,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // Test 1: block with both of those transactions should be rejected. block = CreateAndProcessBlock(spends, scriptPubKey); + LOCK(cs_main); BOOST_CHECK(chainActive.Tip()->GetBlockHash() != block.GetHash()); // Test 2: ... and should be rejected if spend1 is in the memory pool @@ -102,7 +103,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_flags, bool add_to_cache, bool upgraded_nop) +void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_flags, bool add_to_cache) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can @@ -119,29 +120,23 @@ void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_fl // WITNESS requires P2SH test_flags |= SCRIPT_VERIFY_P2SH; } - bool ret = CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, nullptr); + bool ret = CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, nullptr); // CheckInputs should succeed iff test_flags doesn't intersect with // failing_flags bool expected_return_value = !(test_flags & failing_flags); - if (expected_return_value && upgraded_nop) { - // If the script flag being tested corresponds to an upgraded NOP, - // then script execution should fail if DISCOURAGE_UPGRADABLE_NOPS - // is set. - expected_return_value = !(test_flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS); - } BOOST_CHECK_EQUAL(ret, expected_return_value); // Test the caching if (ret && add_to_cache) { // Check that we get a cache hit if the tx was valid std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK(scriptchecks.empty()); } else { // Check that we get script executions to check, if the transaction // was invalid, or we didn't add to cache. std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), tx.vin.size()); } } @@ -151,7 +146,10 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { // Test that passing CheckInputs with one set of script flags doesn't imply // that we would pass again with a different set of flags. - InitScriptExecutionCache(); + { + LOCK(cs_main); + InitScriptExecutionCache(); + } CScript p2pk_scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CScript p2sh_scriptPubKey = GetScriptForDestination(CScriptID(p2pk_scriptPubKey)); @@ -201,20 +199,20 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CValidationState state; PrecomputedTransactionData ptd_spend_tx(spend_tx); - BOOST_CHECK(!CheckInputs(spend_tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); + BOOST_CHECK(!CheckInputs(spend_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); // If we call again asking for scriptchecks (as happens in // ConnectBlock), we should add a script check object for this -- we're // not caching invalidity (if that changes, delete this test case). std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(spend_tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); + BOOST_CHECK(CheckInputs(spend_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), 1); // Test that CheckInputs returns true iff DERSIG-enforcing flags are // not present. Don't add these checks to the cache, so that we can // test later that block validation works fine in the absence of cached // successes. - ValidateCheckInputsForAllFlags(spend_tx, SCRIPT_VERIFY_DERSIG | SCRIPT_VERIFY_LOW_S | SCRIPT_VERIFY_STRICTENC, false, false); + ValidateCheckInputsForAllFlags(spend_tx, SCRIPT_VERIFY_DERSIG | SCRIPT_VERIFY_LOW_S | SCRIPT_VERIFY_STRICTENC, false); // And if we produce a block with this tx, it should be valid (DERSIG not // enabled yet), even though there's no cache entry. @@ -239,7 +237,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) std::vector<unsigned char> vchSig2(p2pk_scriptPubKey.begin(), p2pk_scriptPubKey.end()); invalid_under_p2sh_tx.vin[0].scriptSig << vchSig2; - ValidateCheckInputsForAllFlags(invalid_under_p2sh_tx, SCRIPT_VERIFY_P2SH, true, false); + ValidateCheckInputsForAllFlags(invalid_under_p2sh_tx, SCRIPT_VERIFY_P2SH, true); } // Test CHECKLOCKTIMEVERIFY @@ -262,13 +260,13 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) vchSig.push_back((unsigned char)SIGHASH_ALL); invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 101; - ValidateCheckInputsForAllFlags(invalid_with_cltv_tx, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true); + ValidateCheckInputsForAllFlags(invalid_with_cltv_tx, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true); // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; CValidationState state; PrecomputedTransactionData txdata(invalid_with_cltv_tx); - BOOST_CHECK(CheckInputs(invalid_with_cltv_tx, state, pcoinsTip, true, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputs(invalid_with_cltv_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } // TEST CHECKSEQUENCEVERIFY @@ -290,13 +288,13 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) vchSig.push_back((unsigned char)SIGHASH_ALL); invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 101; - ValidateCheckInputsForAllFlags(invalid_with_csv_tx, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true); + ValidateCheckInputsForAllFlags(invalid_with_csv_tx, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true); // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; CValidationState state; PrecomputedTransactionData txdata(invalid_with_csv_tx); - BOOST_CHECK(CheckInputs(invalid_with_csv_tx, state, pcoinsTip, true, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputs(invalid_with_csv_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } // TODO: add tests for remaining script flags @@ -319,11 +317,11 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) UpdateTransaction(valid_with_witness_tx, 0, sigdata); // This should be valid under all script flags. - ValidateCheckInputsForAllFlags(valid_with_witness_tx, 0, true, false); + ValidateCheckInputsForAllFlags(valid_with_witness_tx, 0, true); // Remove the witness, and check that it is now invalid. valid_with_witness_tx.vin[0].scriptWitness.SetNull(); - ValidateCheckInputsForAllFlags(valid_with_witness_tx, SCRIPT_VERIFY_WITNESS, true, false); + ValidateCheckInputsForAllFlags(valid_with_witness_tx, SCRIPT_VERIFY_WITNESS, true); } { @@ -348,7 +346,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) } // This should be valid under all script flags - ValidateCheckInputsForAllFlags(tx, 0, true, false); + ValidateCheckInputsForAllFlags(tx, 0, true); // Check that if the second input is invalid, but the first input is // valid, the transaction is not cached. @@ -358,12 +356,12 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CValidationState state; PrecomputedTransactionData txdata(tx); // This transaction is now invalid under segwit, because of the second input. - BOOST_CHECK(!CheckInputs(tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); + BOOST_CHECK(!CheckInputs(tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); std::vector<CScriptCheck> scriptchecks; // Make sure this transaction was not cached (ie because the first // input was valid) - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); // Should get 2 script checks back -- caching is on a whole-transaction basis. BOOST_CHECK_EQUAL(scriptchecks.size(), 2); } diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 70d83a2e54..40ef9adb2e 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,10 +1,10 @@ // Copyright (c) 2011-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "arith_uint256.h" -#include "uint256.h" -#include "version.h" -#include "test/test_bitcoin.h" +#include <arith_uint256.h> +#include <uint256.h> +#include <version.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <stdint.h> diff --git a/src/test/univalue_tests.cpp b/src/test/univalue_tests.cpp deleted file mode 100644 index dffe8e55a8..0000000000 --- a/src/test/univalue_tests.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright (c) 2014 BitPay Inc. -// Copyright (c) 2014-2016 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <stdint.h> -#include <vector> -#include <string> -#include <map> -#include <univalue.h> -#include "test/test_bitcoin.h" - -#include <boost/test/unit_test.hpp> - -BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(univalue_constructor) -{ - UniValue v1; - BOOST_CHECK(v1.isNull()); - - UniValue v2(UniValue::VSTR); - BOOST_CHECK(v2.isStr()); - - UniValue v3(UniValue::VSTR, "foo"); - BOOST_CHECK(v3.isStr()); - BOOST_CHECK_EQUAL(v3.getValStr(), "foo"); - - UniValue numTest; - BOOST_CHECK(numTest.setNumStr("82")); - BOOST_CHECK(numTest.isNum()); - BOOST_CHECK_EQUAL(numTest.getValStr(), "82"); - - uint64_t vu64 = 82; - UniValue v4(vu64); - BOOST_CHECK(v4.isNum()); - BOOST_CHECK_EQUAL(v4.getValStr(), "82"); - - int64_t vi64 = -82; - UniValue v5(vi64); - BOOST_CHECK(v5.isNum()); - BOOST_CHECK_EQUAL(v5.getValStr(), "-82"); - - int vi = -688; - UniValue v6(vi); - BOOST_CHECK(v6.isNum()); - BOOST_CHECK_EQUAL(v6.getValStr(), "-688"); - - double vd = -7.21; - UniValue v7(vd); - BOOST_CHECK(v7.isNum()); - BOOST_CHECK_EQUAL(v7.getValStr(), "-7.21"); - - std::string vs("yawn"); - UniValue v8(vs); - BOOST_CHECK(v8.isStr()); - BOOST_CHECK_EQUAL(v8.getValStr(), "yawn"); - - const char *vcs = "zappa"; - UniValue v9(vcs); - BOOST_CHECK(v9.isStr()); - BOOST_CHECK_EQUAL(v9.getValStr(), "zappa"); -} - -BOOST_AUTO_TEST_CASE(univalue_typecheck) -{ - UniValue v1; - BOOST_CHECK(v1.setNumStr("1")); - BOOST_CHECK(v1.isNum()); - BOOST_CHECK_THROW(v1.get_bool(), std::runtime_error); - - UniValue v2; - BOOST_CHECK(v2.setBool(true)); - BOOST_CHECK_EQUAL(v2.get_bool(), true); - BOOST_CHECK_THROW(v2.get_int(), std::runtime_error); - - UniValue v3; - BOOST_CHECK(v3.setNumStr("32482348723847471234")); - BOOST_CHECK_THROW(v3.get_int64(), std::runtime_error); - BOOST_CHECK(v3.setNumStr("1000")); - BOOST_CHECK_EQUAL(v3.get_int64(), 1000); - - UniValue v4; - BOOST_CHECK(v4.setNumStr("2147483648")); - BOOST_CHECK_EQUAL(v4.get_int64(), 2147483648); - BOOST_CHECK_THROW(v4.get_int(), std::runtime_error); - BOOST_CHECK(v4.setNumStr("1000")); - BOOST_CHECK_EQUAL(v4.get_int(), 1000); - BOOST_CHECK_THROW(v4.get_str(), std::runtime_error); - BOOST_CHECK_EQUAL(v4.get_real(), 1000); - BOOST_CHECK_THROW(v4.get_array(), std::runtime_error); - BOOST_CHECK_THROW(v4.getKeys(), std::runtime_error); - BOOST_CHECK_THROW(v4.getValues(), std::runtime_error); - BOOST_CHECK_THROW(v4.get_obj(), std::runtime_error); - - UniValue v5; - BOOST_CHECK(v5.read("[true, 10]")); - BOOST_CHECK_NO_THROW(v5.get_array()); - std::vector<UniValue> vals = v5.getValues(); - BOOST_CHECK_THROW(vals[0].get_int(), std::runtime_error); - BOOST_CHECK_EQUAL(vals[0].get_bool(), true); - - BOOST_CHECK_EQUAL(vals[1].get_int(), 10); - BOOST_CHECK_THROW(vals[1].get_bool(), std::runtime_error); -} - -BOOST_AUTO_TEST_CASE(univalue_set) -{ - UniValue v(UniValue::VSTR, "foo"); - v.clear(); - BOOST_CHECK(v.isNull()); - BOOST_CHECK_EQUAL(v.getValStr(), ""); - - BOOST_CHECK(v.setObject()); - BOOST_CHECK(v.isObject()); - BOOST_CHECK_EQUAL(v.size(), 0); - BOOST_CHECK_EQUAL(v.getType(), UniValue::VOBJ); - BOOST_CHECK(v.empty()); - - BOOST_CHECK(v.setArray()); - BOOST_CHECK(v.isArray()); - BOOST_CHECK_EQUAL(v.size(), 0); - - BOOST_CHECK(v.setStr("zum")); - BOOST_CHECK(v.isStr()); - BOOST_CHECK_EQUAL(v.getValStr(), "zum"); - - BOOST_CHECK(v.setFloat(-1.01)); - BOOST_CHECK(v.isNum()); - BOOST_CHECK_EQUAL(v.getValStr(), "-1.01"); - - BOOST_CHECK(v.setInt((int)1023)); - BOOST_CHECK(v.isNum()); - BOOST_CHECK_EQUAL(v.getValStr(), "1023"); - - BOOST_CHECK(v.setInt((int64_t)-1023LL)); - BOOST_CHECK(v.isNum()); - BOOST_CHECK_EQUAL(v.getValStr(), "-1023"); - - BOOST_CHECK(v.setInt((uint64_t)1023ULL)); - BOOST_CHECK(v.isNum()); - BOOST_CHECK_EQUAL(v.getValStr(), "1023"); - - BOOST_CHECK(v.setNumStr("-688")); - BOOST_CHECK(v.isNum()); - BOOST_CHECK_EQUAL(v.getValStr(), "-688"); - - BOOST_CHECK(v.setBool(false)); - BOOST_CHECK_EQUAL(v.isBool(), true); - BOOST_CHECK_EQUAL(v.isTrue(), false); - BOOST_CHECK_EQUAL(v.isFalse(), true); - BOOST_CHECK_EQUAL(v.getBool(), false); - - BOOST_CHECK(v.setBool(true)); - BOOST_CHECK_EQUAL(v.isBool(), true); - BOOST_CHECK_EQUAL(v.isTrue(), true); - BOOST_CHECK_EQUAL(v.isFalse(), false); - BOOST_CHECK_EQUAL(v.getBool(), true); - - BOOST_CHECK(!v.setNumStr("zombocom")); - - BOOST_CHECK(v.setNull()); - BOOST_CHECK(v.isNull()); -} - -BOOST_AUTO_TEST_CASE(univalue_array) -{ - UniValue arr(UniValue::VARR); - - UniValue v((int64_t)1023LL); - BOOST_CHECK(arr.push_back(v)); - - std::string vStr("zippy"); - BOOST_CHECK(arr.push_back(vStr)); - - const char *s = "pippy"; - BOOST_CHECK(arr.push_back(s)); - - std::vector<UniValue> vec; - v.setStr("boing"); - vec.push_back(v); - - v.setStr("going"); - vec.push_back(v); - - BOOST_CHECK(arr.push_backV(vec)); - - BOOST_CHECK_EQUAL(arr.empty(), false); - BOOST_CHECK_EQUAL(arr.size(), 5); - - BOOST_CHECK_EQUAL(arr[0].getValStr(), "1023"); - BOOST_CHECK_EQUAL(arr[1].getValStr(), "zippy"); - BOOST_CHECK_EQUAL(arr[2].getValStr(), "pippy"); - BOOST_CHECK_EQUAL(arr[3].getValStr(), "boing"); - BOOST_CHECK_EQUAL(arr[4].getValStr(), "going"); - - BOOST_CHECK_EQUAL(arr[999].getValStr(), ""); - - arr.clear(); - BOOST_CHECK(arr.empty()); - BOOST_CHECK_EQUAL(arr.size(), 0); -} - -BOOST_AUTO_TEST_CASE(univalue_object) -{ - UniValue obj(UniValue::VOBJ); - std::string strKey, strVal; - UniValue v; - - strKey = "age"; - v.setInt(100); - BOOST_CHECK(obj.pushKV(strKey, v)); - - strKey = "first"; - strVal = "John"; - BOOST_CHECK(obj.pushKV(strKey, strVal)); - - strKey = "last"; - const char *cVal = "Smith"; - BOOST_CHECK(obj.pushKV(strKey, cVal)); - - strKey = "distance"; - BOOST_CHECK(obj.pushKV(strKey, (int64_t) 25)); - - strKey = "time"; - BOOST_CHECK(obj.pushKV(strKey, (uint64_t) 3600)); - - strKey = "calories"; - BOOST_CHECK(obj.pushKV(strKey, (int) 12)); - - strKey = "temperature"; - BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); - - UniValue obj2(UniValue::VOBJ); - BOOST_CHECK(obj2.pushKV("cat1", 9000)); - BOOST_CHECK(obj2.pushKV("cat2", 12345)); - - BOOST_CHECK(obj.pushKVs(obj2)); - - BOOST_CHECK_EQUAL(obj.empty(), false); - BOOST_CHECK_EQUAL(obj.size(), 9); - - BOOST_CHECK_EQUAL(obj["age"].getValStr(), "100"); - BOOST_CHECK_EQUAL(obj["first"].getValStr(), "John"); - BOOST_CHECK_EQUAL(obj["last"].getValStr(), "Smith"); - BOOST_CHECK_EQUAL(obj["distance"].getValStr(), "25"); - BOOST_CHECK_EQUAL(obj["time"].getValStr(), "3600"); - BOOST_CHECK_EQUAL(obj["calories"].getValStr(), "12"); - BOOST_CHECK_EQUAL(obj["temperature"].getValStr(), "90.012"); - BOOST_CHECK_EQUAL(obj["cat1"].getValStr(), "9000"); - BOOST_CHECK_EQUAL(obj["cat2"].getValStr(), "12345"); - - BOOST_CHECK_EQUAL(obj["nyuknyuknyuk"].getValStr(), ""); - - BOOST_CHECK(obj.exists("age")); - BOOST_CHECK(obj.exists("first")); - BOOST_CHECK(obj.exists("last")); - BOOST_CHECK(obj.exists("distance")); - BOOST_CHECK(obj.exists("time")); - BOOST_CHECK(obj.exists("calories")); - BOOST_CHECK(obj.exists("temperature")); - BOOST_CHECK(obj.exists("cat1")); - BOOST_CHECK(obj.exists("cat2")); - - BOOST_CHECK(!obj.exists("nyuknyuknyuk")); - - std::map<std::string, UniValue::VType> objTypes; - objTypes["age"] = UniValue::VNUM; - objTypes["first"] = UniValue::VSTR; - objTypes["last"] = UniValue::VSTR; - objTypes["distance"] = UniValue::VNUM; - objTypes["time"] = UniValue::VNUM; - objTypes["calories"] = UniValue::VNUM; - objTypes["temperature"] = UniValue::VNUM; - objTypes["cat1"] = UniValue::VNUM; - objTypes["cat2"] = UniValue::VNUM; - BOOST_CHECK(obj.checkObject(objTypes)); - - objTypes["cat2"] = UniValue::VSTR; - BOOST_CHECK(!obj.checkObject(objTypes)); - - obj.clear(); - BOOST_CHECK(obj.empty()); - BOOST_CHECK_EQUAL(obj.size(), 0); -} - -static const char *json1 = -"[1.10000000,{\"key1\":\"str\\u0000\",\"key2\":800,\"key3\":{\"name\":\"martian http://test.com\"}}]"; - -BOOST_AUTO_TEST_CASE(univalue_readwrite) -{ - UniValue v; - BOOST_CHECK(v.read(json1)); - - std::string strJson1(json1); - BOOST_CHECK(v.read(strJson1)); - - BOOST_CHECK(v.isArray()); - BOOST_CHECK_EQUAL(v.size(), 2); - - BOOST_CHECK_EQUAL(v[0].getValStr(), "1.10000000"); - - UniValue obj = v[1]; - BOOST_CHECK(obj.isObject()); - BOOST_CHECK_EQUAL(obj.size(), 3); - - BOOST_CHECK(obj["key1"].isStr()); - std::string correctValue("str"); - correctValue.push_back('\0'); - BOOST_CHECK_EQUAL(obj["key1"].getValStr(), correctValue); - BOOST_CHECK(obj["key2"].isNum()); - BOOST_CHECK_EQUAL(obj["key2"].getValStr(), "800"); - BOOST_CHECK(obj["key3"].isObject()); - - BOOST_CHECK_EQUAL(strJson1, v.write()); - - /* Check for (correctly reporting) a parsing error if the initial - JSON construct is followed by more stuff. Note that whitespace - is, of course, exempt. */ - - BOOST_CHECK(v.read(" {}\n ")); - BOOST_CHECK(v.isObject()); - BOOST_CHECK(v.read(" []\n ")); - BOOST_CHECK(v.isArray()); - - BOOST_CHECK(!v.read("@{}")); - BOOST_CHECK(!v.read("{} garbage")); - BOOST_CHECK(!v.read("[]{}")); - BOOST_CHECK(!v.read("{}[]")); - BOOST_CHECK(!v.read("{} 42")); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6ec544290d..998090cd9b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" - -#include "clientversion.h" -#include "primitives/transaction.h" -#include "sync.h" -#include "utilstrencodings.h" -#include "utilmoneystr.h" -#include "test/test_bitcoin.h" +#include <util.h> + +#include <clientversion.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <utilstrencodings.h> +#include <utilmoneystr.h> +#include <test/test_bitcoin.h> #include <stdint.h> #include <vector> diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 882afb2e20..7029b347dd 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "versionbits.h" -#include "test/test_bitcoin.h" -#include "chainparams.h" -#include "validation.h" -#include "consensus/params.h" +#include <chain.h> +#include <versionbits.h> +#include <test/test_bitcoin.h> +#include <chainparams.h> +#include <validation.h> +#include <consensus/params.h> #include <boost/test/unit_test.hpp> @@ -32,6 +32,12 @@ public: int GetStateSinceHeightFor(const CBlockIndex* pindexPrev) const { return AbstractThresholdConditionChecker::GetStateSinceHeightFor(pindexPrev, paramsDummy, cache); } }; +class TestAlwaysActiveConditionChecker : public TestConditionChecker +{ +public: + int64_t BeginTime(const Consensus::Params& params) const override { return Consensus::BIP9Deployment::ALWAYS_ACTIVE; } +}; + #define CHECKERS 6 class VersionBitsTester @@ -43,6 +49,8 @@ class VersionBitsTester // The first one performs all checks, the second only 50%, the third only 25%, etc... // This is to test whether lack of cached information leads to the same results. TestConditionChecker checker[CHECKERS]; + // Another 6 that assume always active activation + TestAlwaysActiveConditionChecker checker_always[CHECKERS]; // Test counter (to identify failures) int num; @@ -56,6 +64,7 @@ public: } for (unsigned int i = 0; i < CHECKERS; i++) { checker[i] = TestConditionChecker(); + checker_always[i] = TestAlwaysActiveConditionChecker(); } vpblock.clear(); return *this; @@ -82,6 +91,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateSinceHeightFor(vpblock.empty() ? nullptr : vpblock.back()) == height, strprintf("Test %i for StateSinceHeight", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateSinceHeightFor(vpblock.empty() ? nullptr : vpblock.back()) == 0, strprintf("Test %i for StateSinceHeight (always active)", num)); } } num++; @@ -92,6 +102,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_DEFINED, strprintf("Test %i for DEFINED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -102,6 +113,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_STARTED, strprintf("Test %i for STARTED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -112,6 +124,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_LOCKED_IN, strprintf("Test %i for LOCKED_IN", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -122,6 +135,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -132,6 +146,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_FAILED, strprintf("Test %i for FAILED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; diff --git a/src/threadinterrupt.cpp b/src/threadinterrupt.cpp index 9d691079ed..7bcacc3014 100644 --- a/src/threadinterrupt.cpp +++ b/src/threadinterrupt.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "threadinterrupt.h" +#include <threadinterrupt.h> CThreadInterrupt::operator bool() const { diff --git a/src/timedata.cpp b/src/timedata.cpp index 5113bb60db..b91cf01723 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -3,17 +3,17 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "timedata.h" +#include <timedata.h> -#include "netaddress.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" -#include "warnings.h" +#include <netaddress.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> +#include <warnings.h> static CCriticalSection cs_nTimeOffset; diff --git a/src/tinyformat.h b/src/tinyformat.h index 2e453e56bb..d34cfaa94f 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -495,7 +495,11 @@ namespace detail { class FormatArg { public: - FormatArg() {} + FormatArg() + : m_value(nullptr), + m_formatImpl(nullptr), + m_toIntImpl(nullptr) + { } template<typename T> explicit FormatArg(const T& value) @@ -507,11 +511,15 @@ class FormatArg void format(std::ostream& out, const char* fmtBegin, const char* fmtEnd, int ntrunc) const { + assert(m_value); + assert(m_formatImpl); m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); } int toInt() const { + assert(m_value); + assert(m_toIntImpl); return m_toIntImpl(m_value); } @@ -712,23 +720,27 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi break; case 'X': out.setf(std::ios::uppercase); + // Falls through case 'x': case 'p': out.setf(std::ios::hex, std::ios::basefield); intConversion = true; break; case 'E': out.setf(std::ios::uppercase); + // Falls through case 'e': out.setf(std::ios::scientific, std::ios::floatfield); out.setf(std::ios::dec, std::ios::basefield); break; case 'F': out.setf(std::ios::uppercase); + // Falls through case 'f': out.setf(std::ios::fixed, std::ios::floatfield); break; case 'G': out.setf(std::ios::uppercase); + // Falls through case 'g': out.setf(std::ios::dec, std::ios::basefield); // As in boost::format, let stream decide float format. diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 1cea197666..46f39d65a0 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -3,12 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "torcontrol.h" -#include "utilstrencodings.h" -#include "netbase.h" -#include "net.h" -#include "util.h" -#include "crypto/hmac_sha256.h" +#include <torcontrol.h> +#include <utilstrencodings.h> +#include <netbase.h> +#include <net.h> +#include <util.h> +#include <crypto/hmac_sha256.h> #include <vector> #include <deque> diff --git a/src/torcontrol.h b/src/torcontrol.h index 72dc82c5b1..a01eeefeb8 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -8,7 +8,7 @@ #ifndef BITCOIN_TORCONTROL_H #define BITCOIN_TORCONTROL_H -#include "scheduler.h" +#include <scheduler.h> extern const std::string DEFAULT_TOR_CONTROL; static const bool DEFAULT_LISTEN_ONION = true; diff --git a/src/txdb.cpp b/src/txdb.cpp index 134bb8721b..7a8e854b55 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -3,16 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "txdb.h" - -#include "chainparams.h" -#include "hash.h" -#include "random.h" -#include "pow.h" -#include "uint256.h" -#include "util.h" -#include "ui_interface.h" -#include "init.h" +#include <txdb.h> + +#include <chainparams.h> +#include <hash.h> +#include <random.h> +#include <pow.h> +#include <uint256.h> +#include <util.h> +#include <ui_interface.h> +#include <init.h> #include <stdint.h> diff --git a/src/txdb.h b/src/txdb.h index ec9f571b13..2e8095e3cd 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_TXDB_H #define BITCOIN_TXDB_H -#include "coins.h" -#include "dbwrapper.h" -#include "chain.h" +#include <coins.h> +#include <dbwrapper.h> +#include <chain.h> #include <map> #include <string> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index b0306811cb..392323808f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -3,20 +3,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "txmempool.h" - -#include "consensus/consensus.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "validation.h" -#include "policy/policy.h" -#include "policy/fees.h" -#include "reverse_iterator.h" -#include "streams.h" -#include "timedata.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utiltime.h" +#include <txmempool.h> + +#include <consensus/consensus.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <validation.h> +#include <policy/policy.h> +#include <policy/fees.h> +#include <reverse_iterator.h> +#include <streams.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utiltime.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -317,7 +317,7 @@ void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFe assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; assert(int64_t(nSizeWithAncestors) > 0); diff --git a/src/txmempool.h b/src/txmempool.h index 929d223588..86a008d7b2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -13,13 +13,13 @@ #include <utility> #include <string> -#include "amount.h" -#include "coins.h" -#include "indirectmap.h" -#include "policy/feerate.h" -#include "primitives/transaction.h" -#include "sync.h" -#include "random.h" +#include <amount.h> +#include <coins.h> +#include <indirectmap.h> +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <random.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/hashed_index.hpp> @@ -109,7 +109,7 @@ public: // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); + void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(int64_t feeDelta); @@ -204,7 +204,7 @@ struct mempoolentry_txid class CompareTxMemPoolEntryByDescendantScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); @@ -226,7 +226,7 @@ public: } // Calculate which score to use for an entry (avoiding division). - bool UseDescendantScore(const CTxMemPoolEntry &a) + bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = (double)a.GetModifiedFee() * a.GetSizeWithDescendants(); double f2 = (double)a.GetModFeesWithDescendants() * a.GetTxSize(); @@ -241,7 +241,7 @@ public: class CompareTxMemPoolEntryByScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double f1 = (double)a.GetModifiedFee() * b.GetTxSize(); double f2 = (double)b.GetModifiedFee() * a.GetTxSize(); @@ -255,7 +255,7 @@ public: class CompareTxMemPoolEntryByEntryTime { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { return a.GetTime() < b.GetTime(); } @@ -264,7 +264,7 @@ public: class CompareTxMemPoolEntryByAncestorFee { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double aFees = a.GetModFeesWithAncestors(); double aSize = a.GetSizeWithAncestors(); @@ -513,6 +513,9 @@ public: // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. + // Note that addUnchecked is ONLY called from ATMP outside of tests + // and any other callers may break wallet's in-mempool tracking (due to + // lack of CValidationInterface::TransactionAddedToMempool callbacks). bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool validFeeEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate = true); diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 74a13e0e05..c0f7defa46 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "ui_interface.h" -#include "util.h" +#include <ui_interface.h> +#include <util.h> CClientUIInterface uiInterface; diff --git a/src/uint256.cpp b/src/uint256.cpp index 736a0d4fe2..b7255231c1 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -3,9 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "uint256.h" +#include <uint256.h> -#include "utilstrencodings.h" +#include <utilstrencodings.h> #include <stdio.h> #include <string.h> diff --git a/src/uint256.h b/src/uint256.h index 94a4f7fc30..e45c990dae 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -12,14 +12,14 @@ #include <stdint.h> #include <string> #include <vector> -#include "crypto/common.h" +#include <crypto/common.h> /** Template base class for fixed-sized opaque blobs. */ template<unsigned int BITS> class base_blob { protected: - enum { WIDTH=BITS/8 }; + static constexpr int WIDTH = BITS / 8; uint8_t data[WIDTH]; public: base_blob() diff --git a/src/undo.h b/src/undo.h index a720de4ac5..b482fd3089 100644 --- a/src/undo.h +++ b/src/undo.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_UNDO_H #define BITCOIN_UNDO_H -#include "compressor.h" -#include "consensus/consensus.h" -#include "primitives/transaction.h" -#include "serialize.h" +#include <compressor.h> +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <serialize.h> /** Undo information for a CTxIn * diff --git a/src/univalue/.travis.yml b/src/univalue/.travis.yml index 132743d349..43a1ed362e 100644 --- a/src/univalue/.travis.yml +++ b/src/univalue/.travis.yml @@ -25,7 +25,6 @@ addons: - pkg-config before_script: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall libtool; brew install libtool; fi - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 02446292a1..679cc9f143 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -19,9 +19,10 @@ #define BOOST_CHECK_THROW(stmt, excMatch) { \ try { \ (stmt); \ + assert(0 && "No exception caught"); \ } catch (excMatch & e) { \ } catch (...) { \ - assert(0); \ + assert(0 && "Wrong exception caught"); \ } \ } #define BOOST_CHECK_NO_THROW(stmt) { \ diff --git a/src/util.cpp b/src/util.cpp index 51ccc94787..a3e4b0acfb 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -3,18 +3,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#include "util.h" +#include <util.h> -#include "chainparamsbase.h" -#include "fs.h" -#include "random.h" -#include "serialize.h" -#include "utilstrencodings.h" -#include "utiltime.h" +#include <chainparamsbase.h> +#include <random.h> +#include <serialize.h> +#include <utilstrencodings.h> #include <stdarg.h> @@ -89,6 +83,7 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; +const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; ArgsManager gArgs; bool fPrintToConsole = false; @@ -189,26 +184,40 @@ static void DebugPrintInit() vMsgsBeforeOpenLog = new std::list<std::string>; } -void OpenDebugLog() +fs::path GetDebugLogPath() +{ + fs::path logfile(gArgs.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + if (logfile.is_absolute()) { + return logfile; + } else { + return GetDataDir() / logfile; + } +} + +bool OpenDebugLog() { boost::call_once(&DebugPrintInit, debugPrintInitFlag); boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); assert(fileout == nullptr); assert(vMsgsBeforeOpenLog); - fs::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDebugLogPath(); + fileout = fsbridge::fopen(pathDebug, "a"); - if (fileout) { - setbuf(fileout, nullptr); // unbuffered - // dump buffered messages from before we opened the log - while (!vMsgsBeforeOpenLog->empty()) { - FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); - vMsgsBeforeOpenLog->pop_front(); - } + if (!fileout) { + return false; + } + + setbuf(fileout, nullptr); // unbuffered + // dump buffered messages from before we opened the log + while (!vMsgsBeforeOpenLog->empty()) { + FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); + vMsgsBeforeOpenLog->pop_front(); } delete vMsgsBeforeOpenLog; vMsgsBeforeOpenLog = nullptr; + return true; } struct CLogCategoryDesc @@ -220,6 +229,7 @@ struct CLogCategoryDesc const CLogCategoryDesc LogCategories[] = { {BCLog::NONE, "0"}, + {BCLog::NONE, "none"}, {BCLog::NET, "net"}, {BCLog::TOR, "tor"}, {BCLog::MEMPOOL, "mempool"}, @@ -354,7 +364,7 @@ int LogPrintStr(const std::string &str) // reopen the log file, if requested if (fReopenDebugLog) { fReopenDebugLog = false; - fs::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDebugLogPath(); if (fsbridge::freopen(pathDebug,"a",fileout) != nullptr) setbuf(fileout, nullptr); // unbuffered } @@ -574,7 +584,10 @@ const fs::path &GetDataDir(bool fNetSpecific) if (fNetSpecific) path /= BaseParams().DataDir(); - fs::create_directories(path); + if (fs::create_directories(path)) { + // This is the first run, create wallets subdirectory too + fs::create_directories(path / "wallets"); + } return path; } @@ -620,6 +633,9 @@ void ArgsManager::ReadConfigFile(const std::string& confPath) } // If datadir is changed in .conf file: ClearDatadirCache(); + if (!fs::is_directory(GetDataDir(false))) { + throw std::runtime_error(strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str())); + } } #ifndef WIN32 @@ -770,7 +786,7 @@ void ShrinkDebugFile() // Amount of debug.log to save at end when shrinking (must fit in memory) constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; // Scroll debug.log if it's getting too big - fs::path pathLog = GetDataDir() / "debug.log"; + fs::path pathLog = GetDebugLogPath(); FILE* file = fsbridge::fopen(pathLog, "r"); // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes diff --git a/src/util.h b/src/util.h index 480a80c0a3..3cc4c26817 100644 --- a/src/util.h +++ b/src/util.h @@ -11,14 +11,14 @@ #define BITCOIN_UTIL_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "fs.h" -#include "sync.h" -#include "tinyformat.h" -#include "utiltime.h" +#include <compat.h> +#include <fs.h> +#include <sync.h> +#include <tinyformat.h> +#include <utiltime.h> #include <atomic> #include <exception> @@ -28,6 +28,7 @@ #include <vector> #include <boost/signals2/signal.hpp> +#include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted // Application startup time (used for uptime calculation) int64_t GetStartupTime(); @@ -35,6 +36,7 @@ int64_t GetStartupTime(); static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; +extern const char * const DEFAULT_DEBUGLOGFILE; /** Signals for translation. */ class CTranslationInterface @@ -132,6 +134,10 @@ template<typename T, typename... Args> static inline void MarkUsed(const T& t, c MarkUsed(args...); } +// Be conservative when using LogPrintf/error or other things which +// unconditionally log to debug.log! It should not be the case that an inbound +// peer can fill up a users disk with debug.log entries. + #ifdef USE_COVERAGE #define LogPrintf(...) do { MarkUsed(__VA_ARGS__); } while(0) #define LogPrint(category, ...) do { MarkUsed(__VA_ARGS__); } while(0) @@ -179,7 +185,8 @@ void CreatePidFile(const fs::path &path, pid_t pid); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif -void OpenDebugLog(); +fs::path GetDebugLogPath(); +bool OpenDebugLog(); void ShrinkDebugFile(); void runCommand(const std::string& strCommand); @@ -326,4 +333,11 @@ template <typename Callable> void TraceThread(const char* name, Callable func) std::string CopyrightHolders(const std::string& strPrefix); +//! Substitute for C++14 std::make_unique. +template <typename T, typename... Args> +std::unique_ptr<T> MakeUnique(Args&&... args) +{ + return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); +} + #endif // BITCOIN_UTIL_H diff --git a/src/utilmoneystr.cpp b/src/utilmoneystr.cpp index 6e6e33184e..656d7c51b7 100644 --- a/src/utilmoneystr.cpp +++ b/src/utilmoneystr.cpp @@ -3,11 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilmoneystr.h" +#include <utilmoneystr.h> -#include "primitives/transaction.h" -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <primitives/transaction.h> +#include <tinyformat.h> +#include <utilstrencodings.h> std::string FormatMoney(const CAmount& n) { diff --git a/src/utilmoneystr.h b/src/utilmoneystr.h index bc885ee167..b1a4499e90 100644 --- a/src/utilmoneystr.h +++ b/src/utilmoneystr.h @@ -12,7 +12,7 @@ #include <stdint.h> #include <string> -#include "amount.h" +#include <amount.h> /* Do not use these functions to represent or parse monetary amounts to or from * JSON but use AmountFromValue and ValueFromAmount for that. diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 741680e93f..d90713ed6a 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -3,9 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" +#include <utilstrencodings.h> -#include "tinyformat.h" +#include <tinyformat.h> #include <cstdlib> #include <cstring> diff --git a/src/utiltime.cpp b/src/utiltime.cpp index 4cc77dbfeb..ca11cf341f 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -4,10 +4,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "utiltime.h" +#include <utiltime.h> #include <atomic> diff --git a/src/validation.cpp b/src/validation.cpp index a073e3d1c0..75c40b22fc 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3,46 +3,43 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" - -#include "arith_uint256.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "checkqueue.h" -#include "consensus/consensus.h" -#include "consensus/merkle.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "cuckoocache.h" -#include "fs.h" -#include "hash.h" -#include "init.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "pow.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "random.h" -#include "reverse_iterator.h" -#include "script/script.h" -#include "script/sigcache.h" -#include "script/standard.h" -#include "timedata.h" -#include "tinyformat.h" -#include "txdb.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "undo.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "versionbits.h" -#include "warnings.h" - -#include <atomic> +#include <validation.h> + +#include <arith_uint256.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <checkqueue.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <cuckoocache.h> +#include <hash.h> +#include <init.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <pow.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <random.h> +#include <reverse_iterator.h> +#include <script/script.h> +#include <script/sigcache.h> +#include <script/standard.h> +#include <timedata.h> +#include <tinyformat.h> +#include <txdb.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <undo.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <warnings.h> + #include <sstream> #include <boost/algorithm/string/replace.hpp> @@ -59,11 +56,150 @@ /** * Global state */ +namespace { + struct CBlockIndexWorkComparator + { + bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; + } + }; +} // anon namespace + +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + +class ConnectTrace; + +/** + * CChainState stores and provides an API to update our local knowledge of the + * current best chain and header tree. + * + * It generally provides access to the current block tree, as well as functions + * to provide new data, which it will appropriately validate and incorporate in + * its state as necessary. + * + * Eventually, the API here is targeted at being exposed externally as a + * consumable libconsensus library, so any functions added must only call + * other class member functions, pure functions in other parts of the consensus + * library, callbacks via the validation interface, or read/write-to-disk + * functions (eventually this will also be via callbacks). + */ +class CChainState { +private: + /** + * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. + */ + std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; + + /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + CCriticalSection cs_nBlockSequenceId; + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + int32_t nBlockSequenceId = 1; + /** Decreasing counter (used by subsequent preciousblock calls). */ + int32_t nBlockReverseSequenceId = -1; + /** chainwork for the last block that preciousblock has been applied to. */ + arith_uint256 nLastPreciousChainwork = 0; + + /** In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk mapBlockIndex when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of mapBlockIndex to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we already walk mapBlockIndex in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set<CBlockIndex*> g_failed_blocks; + +public: + CChain chainActive; + BlockMap mapBlockIndex; + std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; + CBlockIndex *pindexBestInvalid = nullptr; + + bool LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree); + + bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock); + + bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock); + + // Block (dis)connection on a given view: + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false); + + // Block disconnection on our pcoinsTip: + bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool); + + // Manual block validity manipulation: + bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex); + bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex); + bool ResetBlockFailureFlags(CBlockIndex *pindex); + + bool ReplayBlocks(const CChainParams& params, CCoinsView* view); + bool RewindBlockIndex(const CChainParams& params); + bool LoadGenesisBlock(const CChainParams& chainparams); + + void PruneBlockIndexCandidates(); + + void UnloadBlockIndex(); + +private: + bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace); + bool ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool); + + CBlockIndex* AddToBlockIndex(const CBlockHeader& block); + /** Create a new block index entry for a given block hash */ + CBlockIndex * InsertBlockIndex(const uint256& hash); + void CheckBlockIndex(const Consensus::Params& consensusParams); + + void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state); + CBlockIndex* FindMostWorkChain(); + bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); + + + bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params); +} g_chainstate; + + CCriticalSection cs_main; -BlockMap mapBlockIndex; -CChain chainActive; +BlockMap& mapBlockIndex = g_chainstate.mapBlockIndex; +CChain& chainActive = g_chainstate.chainActive; CBlockIndex *pindexBestHeader = nullptr; CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; @@ -91,8 +227,6 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); -static void CheckBlockIndex(const Consensus::Params& consensusParams); - /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; @@ -100,40 +234,12 @@ const std::string strMessageMagic = "Bitcoin Signed Message:\n"; // Internal stuff namespace { + CBlockIndex *&pindexBestInvalid = g_chainstate.pindexBestInvalid; - struct CBlockIndexWorkComparator - { - bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; - } - }; - - CBlockIndex *pindexBestInvalid; - - /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. - */ - std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; /** All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. */ - std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; + std::multimap<CBlockIndex*, CBlockIndex*>& mapBlocksUnlinked = g_chainstate.mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; @@ -144,18 +250,6 @@ namespace { */ bool fCheckForPruning = false; - /** - * Every received block is assigned a unique and increasing identifier, so we - * know which one to give priority in case of a fork. - */ - CCriticalSection cs_nBlockSequenceId; - /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId = 1; - /** Decreasing counter (used by subsequent preciousblock calls). */ - int32_t nBlockReverseSequenceId = -1; - /** chainwork for the last block that preciousblock has been applied to. */ - arith_uint256 nLastPreciousChainwork = 0; - /** Dirty block index entries. */ std::set<CBlockIndex*> setDirtyBlockIndex; @@ -181,9 +275,9 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc return chain.Genesis(); } -CCoinsViewDB *pcoinsdbview = nullptr; -CCoinsViewCache *pcoinsTip = nullptr; -CBlockTreeDB *pblocktree = nullptr; +std::unique_ptr<CCoinsViewDB> pcoinsdbview; +std::unique_ptr<CCoinsViewCache> pcoinsTip; +std::unique_ptr<CBlockTreeDB> pblocktree; enum FlushStateMode { FLUSH_STATE_NONE, @@ -275,7 +369,7 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool } else { // pcoinsTip contains the UTXO set for chainActive.Tip() - CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); + CCoinsViewMemPool viewMemPool(pcoinsTip.get(), mempool); std::vector<int> prevheights; prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { @@ -404,7 +498,7 @@ void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool f mempool.UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + mempool.removeForReorg(pcoinsTip.get(), chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); } @@ -537,7 +631,7 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool LockPoints lp; { LOCK(pool.cs); - CCoinsViewMemPool viewMemPool(pcoinsTip, pool); + CCoinsViewMemPool viewMemPool(pcoinsTip.get(), pool); view.SetBackend(viewMemPool); // do all inputs exist? @@ -906,44 +1000,51 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, pfMissingInputs, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee); } -/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +/** + * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. + * If blockIndex is provided, the transaction is fetched from the corresponding block. + */ +bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, bool fAllowSlow, CBlockIndex* blockIndex) { - CBlockIndex *pindexSlow = nullptr; + CBlockIndex* pindexSlow = blockIndex; LOCK(cs_main); - CTransactionRef ptx = mempool.get(hash); - if (ptx) - { - txOut = ptx; - return true; - } + if (!blockIndex) { + CTransactionRef ptx = mempool.get(hash); + if (ptx) { + txOut = ptx; + return true; + } - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - if (file.IsNull()) - return error("%s: OpenBlockFile failed", __func__); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + if (fTxIndex) { + CDiskTxPos postx; + if (pblocktree->ReadTxIndex(hash, postx)) { + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> txOut; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + hashBlock = header.GetHash(); + if (txOut->GetHash() != hash) + return error("%s: txid mismatch", __func__); + return true; } - hashBlock = header.GetHash(); - if (txOut->GetHash() != hash) - return error("%s: txid mismatch", __func__); - return true; + + // transaction not found in index, nothing more can be done + return false; } - } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - const Coin& coin = AccessByTxid(*pcoinsTip, hash); - if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; + } } if (pindexSlow) { @@ -1174,9 +1275,10 @@ void static InvalidChainFound(CBlockIndex* pindexNew) CheckForkWarningConditions(); } -void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; + g_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -1356,8 +1458,13 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uint256& hashBlock) +static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) { + CDiskBlockPos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + // Open history file to read CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) @@ -1367,7 +1474,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin uint256 hashChecksum; CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - verifier << hashBlock; + verifier << pindex->pprev->GetBlockHash(); verifier >> blockundo; filein >> hashChecksum; } @@ -1402,13 +1509,6 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // namespace -enum DisconnectResult -{ - DISCONNECT_OK, // All good. - DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. - DISCONNECT_FAILED // Something else went wrong. -}; - /** * Restore the UTXO in a Coin at a given COutPoint * @param undo The Coin to be restored. @@ -1445,17 +1545,12 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When FAILED is returned, view is left in an indeterminate state. */ -static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { bool fClean = true; CBlockUndo blockUndo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - error("DisconnectBlock(): no undo data available"); - return DISCONNECT_FAILED; - } - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + if (!UndoReadFromDisk(blockUndo, pindex)) { error("DisconnectBlock(): failure reading undo data"); return DISCONNECT_FAILED; } @@ -1532,6 +1627,45 @@ void static FlushBlockFile(bool fFinalize = false) static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + CDiskBlockPos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) + return AbortNode(state, "Failed to write undo data"); + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} + +static bool WriteTxIndexDataForBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex) +{ + if (!fTxIndex) return true; + + CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + std::vector<std::pair<uint256, CDiskTxPos> > vPos; + vPos.reserve(block.vtx.size()); + for (const CTransactionRef& tx : block.vtx) + { + vPos.push_back(std::make_pair(tx->GetHash(), pos)); + pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + } + + if (!pblocktree->WriteTxIndex(vPos)) { + return AbortNode(state, "Failed to write transaction index"); + } + + return true; +} + static CCheckQueue<CScriptCheck> scriptcheckqueue(128); void ThreadScriptCheck() { @@ -1587,11 +1721,12 @@ static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS]; static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) { AssertLockHeld(cs_main); - // BIP16 didn't become active until Apr 1 2012 - int64_t nBIP16SwitchTime = 1333238400; - bool fStrictPayToScriptHash = (pindex->GetBlockTime() >= nBIP16SwitchTime); + unsigned int flags = SCRIPT_VERIFY_NONE; - unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; + // Start enforcing P2SH (BIP16) + if (pindex->nHeight >= consensusparams.BIP16Height) { + flags |= SCRIPT_VERIFY_P2SH; + } // Start enforcing the DERSIG (BIP66) rule if (pindex->nHeight >= consensusparams.BIP66Height) { @@ -1631,8 +1766,8 @@ static int64_t nBlocksTotal = 0; /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ -static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -1642,6 +1777,18 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in + // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or + // ContextualCheckBlockHeader() here. This means that if we add a new + // consensus rule that is enforced in one of those two functions, then we + // may have let in a block that violates the rule prior to updating the + // software, and we would NOT be enforcing the rule here. Fully solving + // upgrade from one software version to the next after a consensus rule + // change is potentially tricky and issue-specific (see RewindBlockIndex() + // for one general approach that was used for BIP 141 deployment). + // Also, currently the rule against blocks more than 2 hours in the future + // is enforced in ContextualCheckBlockHeader(); we wouldn't want to + // re-enforce that rule here (at least until we make it impossible for + // GetAdjustedTime() to go backward). if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); @@ -1746,9 +1893,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector<std::pair<uint256, CDiskTxPos> > vPos; - vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector<PrecomputedTransactionData> txdata; txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated @@ -1809,9 +1953,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd blockundo.vtxundo.push_back(CTxUndo()); } UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); - - vPos.push_back(std::make_pair(tx.GetHash(), pos)); - pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); @@ -1831,28 +1972,16 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (fJustCheck) return true; - // Write undo information to disk - if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) - { - if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - } + if (!WriteUndoDataForBlock(blockundo, state, pindex, chainparams)) + return false; + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); setDirtyBlockIndex.insert(pindex); } - if (fTxIndex) - if (!pblocktree->WriteTxIndex(vPos)) - return AbortNode(state, "Failed to write transaction index"); + if (!WriteTxIndexDataForBlock(block, state, pindex)) + return false; assert(pindex->phashBlock); // add this block to the view's block chain @@ -2004,10 +2133,8 @@ static void DoWarning(const std::string& strWarning) } } -/** Update chainActive and related internal data structures. */ -void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { - chainActive.SetTip(pindexNew); - +/** Check warning conditions and do some notifications on new chain tip set. */ +void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainParams) { // New best block mempool.AddTransactionsUpdated(1); @@ -2017,7 +2144,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { if (!IsInitialBlockDownload()) { int nUpgraded = 0; - const CBlockIndex* pindex = chainActive.Tip(); + const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); @@ -2048,10 +2175,10 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, - log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, + log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime()), + GuessVerificationProgress(chainParams.TxData(), pindexNew), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); if (!warningMessages.empty()) LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); LogPrintf("\n"); @@ -2068,7 +2195,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) +bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2080,7 +2207,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { - CCoinsViewCache view(pcoinsTip); + CCoinsViewCache view(pcoinsTip.get()); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); @@ -2105,7 +2232,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara } } - // Update chainActive and related variables. + chainActive.SetTip(pindexDelete->pprev); + UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: @@ -2190,7 +2318,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2210,7 +2338,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { - CCoinsViewCache view(pcoinsTip); + CCoinsViewCache view(pcoinsTip.get()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { @@ -2234,6 +2362,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. + chainActive.SetTip(pindexNew); UpdateTip(pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; @@ -2248,7 +2377,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, * Return the tip of the chain with the most work in it, that isn't * known to be invalid (it's however far from certain to be valid). */ -static CBlockIndex* FindMostWorkChain() { +CBlockIndex* CChainState::FindMostWorkChain() { do { CBlockIndex *pindexNew = nullptr; @@ -2303,7 +2432,7 @@ static CBlockIndex* FindMostWorkChain() { } /** Delete all entries in setBlockIndexCandidates that are worse than the current tip. */ -static void PruneBlockIndexCandidates() { +void CChainState::PruneBlockIndexCandidates() { // Note that we can't delete the current block itself, as we may need to return to it later in case a // reorganization to a better block fails. std::set<CBlockIndex*, CBlockIndexWorkComparator>::iterator it = setBlockIndexCandidates.begin(); @@ -2318,7 +2447,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either nullptr or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -2388,7 +2517,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // any disconnected transactions back to the mempool. UpdateMempoolForReorg(disconnectpool, true); } - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2425,7 +2554,7 @@ static void NotifyHeaderTip() { * or an activated best chain. pblock is either nullptr or a pointer to a block * that is already loaded (to avoid loading it again from disk). */ -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set @@ -2469,7 +2598,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); - GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2495,9 +2624,11 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return true; } +bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { + return g_chainstate.ActivateBestChain(state, chainparams, std::move(pblock)); +} - -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { LOCK(cs_main); @@ -2523,24 +2654,28 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn } } - return ActivateBestChain(state, params); + return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); +} +bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { + return g_chainstate.PreciousBlock(state, params, pindex); } -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { AssertLockHeld(cs_main); - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); + // We first disconnect backwards and then mark the blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + + bool pindex_was_in_chain = false; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { - CBlockIndex *pindexWalk = chainActive.Tip(); - pindexWalk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindexWalk); - setBlockIndexCandidates.erase(pindexWalk); + pindex_was_in_chain = true; // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. if (!DisconnectTip(state, chainparams, &disconnectpool)) { @@ -2551,6 +2686,21 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } } + // Now mark the blocks we just disconnected as descendants invalid + // (note this may not be all descendants). + while (pindex_was_in_chain && invalid_walk_tip != pindex) { + invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + invalid_walk_tip = invalid_walk_tip->pprev; + } + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + g_failed_blocks.insert(pindex); + // DisconnectTip will add transactions to disconnectpool; try to add these // back to the mempool. UpdateMempoolForReorg(disconnectpool, true); @@ -2569,8 +2719,11 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { + return g_chainstate.InvalidateBlock(state, chainparams, pindex); +} -bool ResetBlockFailureFlags(CBlockIndex *pindex) { +bool CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; @@ -2588,6 +2741,7 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { // Reset invalid block marker if it was pointing to one of those. pindexBestInvalid = nullptr; } + g_failed_blocks.erase(it->second); } it++; } @@ -2602,8 +2756,11 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { } return true; } +bool ResetBlockFailureFlags(CBlockIndex *pindex) { + return g_chainstate.ResetBlockFailureFlags(pindex); +} -static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) +CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) { // Check for duplicate uint256 hash = block.GetHash(); @@ -2613,7 +2770,6 @@ static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) // Construct new block index object CBlockIndex* pindexNew = new CBlockIndex(block); - assert(pindexNew); // We assign the sequence id to blocks only when the full data is available, // to avoid miners withholding blocks but broadcasting headers, to get a // competitive advantage. @@ -2639,7 +2795,7 @@ static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool CChainState::ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2687,7 +2843,7 @@ static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& sta return true; } -static bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -2736,7 +2892,7 @@ static bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned i } } else - return state.Error("out of disk space"); + return error("out of disk space"); } } @@ -2911,7 +3067,13 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO - * set; UTXO-related validity checks are done in ConnectBlock(). */ + * set; UTXO-related validity checks are done in ConnectBlock(). + * NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) { assert(pindexPrev != nullptr); @@ -2951,6 +3113,12 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta return true; } +/** NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; @@ -3032,7 +3200,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c return true; } -static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate @@ -3064,6 +3232,21 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + for (const CBlockIndex* failedit : g_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } + } + } } if (pindex == nullptr) pindex = AddToBlockIndex(block); @@ -3077,13 +3260,15 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) { + if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - if (!AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (!g_chainstate.AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (first_invalid) *first_invalid = header; return false; } if (ppindex) { @@ -3096,7 +3281,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + if (dbp != nullptr) + blockPos = *dbp; + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { + error("%s: FindBlockPos failed", __func__); + return CDiskBlockPos(); + } + if (dbp == nullptr) { + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + AbortNode("Failed to write block"); + return CDiskBlockPos(); + } + } + return blockPos; +} + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3113,7 +3317,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (chainActive.Tip() ? pindex->nChainWork >= chainActive.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test @@ -3130,9 +3334,15 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation // and unrequested blocks. if (fAlreadyHave) return true; if (!fRequested) { // If we didn't ask for it: - if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned - if (!fHasMoreWork) return true; // Don't process less-work chains - if (fTooFarAhead) return true; // Block height is too high + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + + // Protect against DoS attacks from low-work chains. + // If our tip is behind, a peer could try to send us + // low-work blocks on a fake chain that we would never + // request; don't process these. + if (pindex->nChainWork < nMinimumChainWork) return true; } if (fNewBlock) *fNewBlock = true; @@ -3150,19 +3360,13 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (!IsInitialBlockDownload() && chainActive.Tip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); - int nHeight = pindex->nHeight; - // Write block to history file try { - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - if (dbp != nullptr) - blockPos = *dbp; - if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) - return error("AcceptBlock(): FindBlockPos failed"); - if (dbp == nullptr) - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) - AbortNode(state, "Failed to write block"); + CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + if (blockPos.IsNull()) { + state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); + return false; + } if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { @@ -3172,6 +3376,8 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (fCheckForPruning) FlushStateToDisk(chainparams, state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + CheckBlockIndex(chainparams.GetConsensus()); + return true; } @@ -3189,9 +3395,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons if (ret) { // Store to disk - ret = AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); + ret = g_chainstate.AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } - CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { GetMainSignals().BlockChecked(*pblock, state); return error("%s: AcceptBlock FAILED (%s)", __func__, state.GetDebugMessage()); @@ -3201,7 +3406,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons NotifyHeaderTip(); CValidationState state; // Only used to report errors, not invalidity - ignore it - if (!ActivateBestChain(state, chainparams, pblock)) + if (!g_chainstate.ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed", __func__); return true; @@ -3211,7 +3416,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, { AssertLockHeld(cs_main); assert(pindexPrev && pindexPrev == chainActive.Tip()); - CCoinsViewCache viewNew(pcoinsTip); + CCoinsViewCache viewNew(pcoinsTip.get()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; @@ -3223,7 +3428,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!g_chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -3251,8 +3456,8 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { - CBlockIndex* pindex = it->second; + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; pindex->nStatus &= ~BLOCK_HAVE_UNDO; @@ -3400,7 +3605,7 @@ static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fRe return nullptr; fs::path path = GetBlockPosFilename(pos, prefix); fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, "rb+"); + FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); if (!file && !fReadOnly) file = fsbridge::fopen(path, "wb+"); if (!file) { @@ -3431,7 +3636,7 @@ fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); } -CBlockIndex * InsertBlockIndex(uint256 hash) +CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) { if (hash.IsNull()) return nullptr; @@ -3443,17 +3648,15 @@ CBlockIndex * InsertBlockIndex(uint256 hash) // Create new CBlockIndex* pindexNew = new CBlockIndex(); - if (!pindexNew) - throw std::runtime_error(std::string(__func__) + ": new CBlockIndex failed"); mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); return pindexNew; } -bool static LoadBlockIndexDB(const CChainParams& chainparams) +bool CChainState::LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree) { - if (!pblocktree->LoadBlockIndexGuts(chainparams.GetConsensus(), InsertBlockIndex)) + if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash){ return this->InsertBlockIndex(hash); })) return false; boost::this_thread::interruption_point(); @@ -3486,6 +3689,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindex->nChainTx = pindex->nTx; } } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -3496,6 +3703,14 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindexBestHeader = pindex; } + return true; +} + +bool static LoadBlockIndexDB(const CChainParams& chainparams) +{ + if (!g_chainstate.LoadBlockIndex(chainparams.GetConsensus(), *pblocktree)) + return false; + // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); @@ -3568,7 +3783,7 @@ bool LoadChainTip(const CChainParams& chainparams) return false; chainActive.SetTip(it->second); - PruneBlockIndexCandidates(); + g_chainstate.PruneBlockIndexCandidates(); LogPrintf("Loaded best chain: hashBestChain=%s height=%d date=%s progress=%f\n", chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), @@ -3633,16 +3848,16 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (!pos.IsNull()) { - if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) + if (!pindex->GetUndoPos().IsNull()) { + if (!UndoReadFromDisk(undo, pindex)) { return error("VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + } } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); - DisconnectResult res = DisconnectBlock(block, pindex, coins); + DisconnectResult res = g_chainstate.DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -3670,7 +3885,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins, chainparams)) + if (!g_chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } @@ -3682,7 +3897,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } /** Apply the effects of a block on the utxo cache, ignoring that it may already have been applied. */ -static bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) +bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) { // TODO: merge with ConnectBlock CBlock block; @@ -3702,7 +3917,7 @@ static bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, return true; } -bool ReplayBlocks(const CChainParams& params, CCoinsView* view) +bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view) { LOCK(cs_main); @@ -3767,7 +3982,11 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) return true; } -bool RewindBlockIndex(const CChainParams& params) +bool ReplayBlocks(const CChainParams& params, CCoinsView* view) { + return g_chainstate.ReplayBlocks(params, view); +} + +bool CChainState::RewindBlockIndex(const CChainParams& params) { LOCK(cs_main); @@ -3804,8 +4023,8 @@ bool RewindBlockIndex(const CChainParams& params) // Reduce validity flag and have-data flags. // We do this after actual disconnecting, otherwise we'll end up writing the lack of data // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - CBlockIndex* pindexIter = it->second; + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindexIter = entry.second; // Note: If we encounter an insufficiently validated block that // is on chainActive, it must be because we are a pruning node, and @@ -3848,10 +4067,21 @@ bool RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); + } + + return true; +} +bool RewindBlockIndex(const CChainParams& params) { + if (!g_chainstate.RewindBlockIndex(params)) { + return false; + } + + if (chainActive.Tip() != nullptr) { // FlushStateToDisk can possibly read chainActive. Be conservative // and skip it here, we're about to -reindex-chainstate anyway, so // it'll get called a bunch real soon. + CValidationState state; if (!FlushStateToDisk(params, state, FLUSH_STATE_ALWAYS)) { return false; } @@ -3860,13 +4090,18 @@ bool RewindBlockIndex(const CChainParams& params) return true; } +void CChainState::UnloadBlockIndex() { + nBlockSequenceId = 1; + g_failed_blocks.clear(); + setBlockIndexCandidates.clear(); +} + // May NOT be used after any connections are up as much // of the peer-processing logic assumes a consistent // block index state void UnloadBlockIndex() { LOCK(cs_main); - setBlockIndexCandidates.clear(); chainActive.SetTip(nullptr); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; @@ -3874,7 +4109,6 @@ void UnloadBlockIndex() mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; - nBlockSequenceId = 1; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); @@ -3887,6 +4121,8 @@ void UnloadBlockIndex() } mapBlockIndex.clear(); fHavePruned = false; + + g_chainstate.UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -3914,7 +4150,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) return true; } -bool LoadGenesisBlock(const CChainParams& chainparams) +bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) { LOCK(cs_main); @@ -3927,15 +4163,11 @@ bool LoadGenesisBlock(const CChainParams& chainparams) try { CBlock &block = const_cast<CBlock&>(chainparams.GenesisBlock()); - // Start new block file - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - CValidationState state; - if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.GetBlockTime())) - return error("%s: FindBlockPos failed", __func__); - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); + CValidationState state; if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("%s: genesis block not accepted", __func__); } catch (const std::runtime_error& e) { @@ -3945,6 +4177,11 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return true; } +bool LoadGenesisBlock(const CChainParams& chainparams) +{ + return g_chainstate.LoadGenesisBlock(chainparams); +} + bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) @@ -4005,7 +4242,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { LOCK(cs_main); CValidationState state; - if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) + if (g_chainstate.AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) nLoaded++; if (state.IsError()) break; @@ -4039,7 +4276,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB head.ToString()); LOCK(cs_main); CValidationState dummy; - if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) + if (g_chainstate.AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -4062,7 +4299,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB return nLoaded > 0; } -void static CheckBlockIndex(const Consensus::Params& consensusParams) +void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) { if (!fCheckBlockIndex) { return; @@ -4080,8 +4317,8 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) // Build forward-pointing map of the entire block tree. std::multimap<CBlockIndex*,CBlockIndex*> forward; - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - forward.insert(std::make_pair(it->second->pprev, it->second)); + for (auto& entry : mapBlockIndex) { + forward.insert(std::make_pair(entry.second->pprev, entry.second)); } assert(forward.size() == mapBlockIndex.size()); @@ -4290,8 +4527,9 @@ bool LoadMempool(void) } int64_t count = 0; - int64_t skipped = 0; + int64_t expired = 0; int64_t failed = 0; + int64_t already_there = 0; int64_t nNow = GetTime(); try { @@ -4322,10 +4560,18 @@ bool LoadMempool(void) if (state.IsValid()) { ++count; } else { - ++failed; + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (mempool.exists(tx->GetHash())) { + ++already_there; + } else { + ++failed; + } } } else { - ++skipped; + ++expired; } if (ShutdownRequested()) return false; @@ -4341,7 +4587,7 @@ bool LoadMempool(void) return false; } - LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there\n", count, failed, expired, already_there); return true; } @@ -4395,7 +4641,7 @@ bool DumpMempool(void) } //! Guess how far we are in the verification process at the given block index -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) { +double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { if (pindex == nullptr) return 0.0; diff --git a/src/validation.h b/src/validation.h index 6bc52753c5..4c8e041af0 100644 --- a/src/validation.h +++ b/src/validation.h @@ -7,17 +7,17 @@ #define BITCOIN_VALIDATION_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "amount.h" -#include "coins.h" -#include "fs.h" -#include "protocol.h" // For CMessageHeader::MessageStartChars -#include "policy/feerate.h" -#include "script/script_error.h" -#include "sync.h" -#include "versionbits.h" +#include <amount.h> +#include <coins.h> +#include <fs.h> +#include <protocol.h> // For CMessageHeader::MessageStartChars +#include <policy/feerate.h> +#include <script/script_error.h> +#include <sync.h> +#include <versionbits.h> #include <algorithm> #include <exception> @@ -159,7 +159,7 @@ extern CCriticalSection cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; -extern BlockMap mapBlockIndex; +extern BlockMap& mapBlockIndex; extern uint64_t nLastBlockTx; extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; @@ -203,6 +203,8 @@ extern bool fPruneMode; extern uint64_t nPruneTarget; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static const unsigned int DEFAULT_CHECKLEVEL = 3; @@ -247,8 +249,9 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons * @param[out] state This may be set to an Error state if any error occurred processing them * @param[in] chainparams The params for the chain we want to connect to * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers + * @param[out] first_invalid First header that fails validation, if one exists */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr); +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); @@ -272,13 +275,13 @@ void ThreadScriptCheck(); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); +bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, bool fAllowSlow = false, CBlockIndex* blockIndex = nullptr); /** Find the best known block, and make it the tip of the block chain */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex* pindex); +double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); @@ -293,8 +296,6 @@ void PruneOneBlockFile(const int fileNumber); */ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); -/** Create a new block index entry for a given block hash */ -CBlockIndex * InsertBlockIndex(uint256 hash); /** Flush all state, indexes and buffers to disk. */ void FlushStateToDisk(); /** Prune block files and flush state to disk. */ @@ -440,16 +441,16 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C bool ResetBlockFailureFlags(CBlockIndex *pindex); /** The currently-connected chain of blocks (protected by cs_main). */ -extern CChain chainActive; +extern CChain& chainActive; /** Global variable that points to the coins database (protected by cs_main) */ -extern CCoinsViewDB *pcoinsdbview; +extern std::unique_ptr<CCoinsViewDB> pcoinsdbview; /** Global variable that points to the active CCoinsView (protected by cs_main) */ -extern CCoinsViewCache *pcoinsTip; +extern std::unique_ptr<CCoinsViewCache> pcoinsTip; /** Global variable that points to the active block tree (protected by cs_main) */ -extern CBlockTreeDB *pblocktree; +extern std::unique_ptr<CBlockTreeDB> pblocktree; /** * Return the spend height, which is one more than the inputs.GetBestBlock(). diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 281bc04b0a..abbd8cc4d2 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -3,13 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validationinterface.h" +#include <validationinterface.h> -#include "init.h" -#include "primitives/block.h" -#include "scheduler.h" -#include "sync.h" -#include "util.h" +#include <init.h> +#include <primitives/block.h> +#include <scheduler.h> +#include <sync.h> +#include <txmempool.h> +#include <util.h> #include <list> #include <atomic> @@ -21,6 +22,7 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; boost::signals2::signal<void (const uint256 &)> Inventory; boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; @@ -47,7 +49,17 @@ void CMainSignals::UnregisterBackgroundSignalScheduler() { } void CMainSignals::FlushBackgroundCallbacks() { - m_internals->m_schedulerClient.EmptyQueue(); + if (m_internals) { + m_internals->m_schedulerClient.EmptyQueue(); + } +} + +void CMainSignals::RegisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); +} + +void CMainSignals::UnregisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.disconnect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); } CMainSignals& GetMainSignals() @@ -60,6 +72,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.connect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.m_internals->Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.m_internals->Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -75,11 +88,15 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.m_internals->NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { + if (!g_signals.m_internals) { + return; + } g_signals.m_internals->BlockChecked.disconnect_all_slots(); g_signals.m_internals->Broadcast.disconnect_all_slots(); g_signals.m_internals->Inventory.disconnect_all_slots(); @@ -87,32 +104,57 @@ void UnregisterAllValidationInterfaces() { g_signals.m_internals->TransactionAddedToMempool.disconnect_all_slots(); g_signals.m_internals->BlockConnected.disconnect_all_slots(); g_signals.m_internals->BlockDisconnected.disconnect_all_slots(); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect_all_slots(); g_signals.m_internals->UpdatedBlockTip.disconnect_all_slots(); g_signals.m_internals->NewPoWValidBlock.disconnect_all_slots(); } +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { + g_signals.m_internals->m_schedulerClient.AddToProcessQueue(std::move(func)); +} + +void CMainSignals::MempoolEntryRemoved(CTransactionRef ptx, MemPoolRemovalReason reason) { + if (reason != MemPoolRemovalReason::BLOCK && reason != MemPoolRemovalReason::CONFLICT) { + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionRemovedFromMempool(ptx); + }); + } +} + void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + m_internals->m_schedulerClient.AddToProcessQueue([pindexNew, pindexFork, fInitialDownload, this] { + m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + }); } void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - m_internals->TransactionAddedToMempool(ptx); + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionAddedToMempool(ptx); + }); } -void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { - m_internals->BlockConnected(pblock, pindex, vtxConflicted); +void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>>& pvtxConflicted) { + m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, pvtxConflicted, this] { + m_internals->BlockConnected(pblock, pindex, *pvtxConflicted); + }); } void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->BlockDisconnected(pblock); + m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { + m_internals->BlockDisconnected(pblock); + }); } void CMainSignals::SetBestChain(const CBlockLocator &locator) { - m_internals->SetBestChain(locator); + m_internals->m_schedulerClient.AddToProcessQueue([locator, this] { + m_internals->SetBestChain(locator); + }); } void CMainSignals::Inventory(const uint256 &hash) { - m_internals->Inventory(hash); + m_internals->m_schedulerClient.AddToProcessQueue([hash, this] { + m_internals->Inventory(hash); + }); } void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { diff --git a/src/validationinterface.h b/src/validationinterface.h index d6da2bc1fd..7b5d138414 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -6,9 +6,10 @@ #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H -#include <memory> +#include <primitives/transaction.h> // CTransaction(Ref) -#include "primitives/transaction.h" // CTransaction(Ref) +#include <functional> +#include <memory> class CBlock; class CBlockIndex; @@ -20,6 +21,8 @@ class CValidationInterface; class CValidationState; class uint256; class CScheduler; +class CTxMemPool; +enum class MemPoolRemovalReason; // These functions dispatch to one or all registered wallets @@ -29,23 +32,66 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn); void UnregisterValidationInterface(CValidationInterface* pwalletIn); /** Unregister all wallets from core */ void UnregisterAllValidationInterfaces(); +/** + * Pushes a function to callback onto the notification queue, guaranteeing any + * callbacks generated prior to now are finished when the function is called. + * + * Be very careful blocking on func to be called if any locks are held - + * validation interface clients may not be able to make progress as they often + * wait for things like cs_main, so blocking until func is called with cs_main + * will result in a deadlock (that DEBUG_LOCKORDER will miss). + */ +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func); class CValidationInterface { protected: - /** Notifies listeners of updated block chain tip */ + /** + * Notifies listeners of updated block chain tip + * + * Called on a background thread. + */ virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - /** Notifies listeners of a transaction having been added to mempool. */ + /** + * Notifies listeners of a transaction having been added to mempool. + * + * Called on a background thread. + */ virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} /** + * Notifies listeners of a transaction leaving mempool. + * + * This only fires for transactions which leave mempool because of expiry, + * size limiting, reorg (changes in lock times/coinbase maturity), or + * replacement. This does not include any transactions which are included + * in BlockConnectedDisconnected either in block->vtx or in txnConflicted. + * + * Called on a background thread. + */ + virtual void TransactionRemovedFromMempool(const CTransactionRef &ptx) {} + /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. + * + * Called on a background thread. */ virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} - /** Notifies listeners of a block being disconnected */ + /** + * Notifies listeners of a block being disconnected + * + * Called on a background thread. + */ virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} - /** Notifies listeners of the new active block chain on-disk. */ + /** + * Notifies listeners of the new active block chain on-disk. + * + * Called on a background thread. + */ virtual void SetBestChain(const CBlockLocator &locator) {} - /** Notifies listeners about an inventory item being seen on the network. */ + /** + * Notifies listeners about an inventory item being seen on the network. + * + * Called on a background thread. + */ virtual void Inventory(const uint256 &hash) {} /** Tells listeners to broadcast their data. */ virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} @@ -73,6 +119,9 @@ private: friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); friend void ::UnregisterAllValidationInterfaces(); + friend void ::CallFunctionInValidationInterfaceQueue(std::function<void ()> func); + + void MempoolEntryRemoved(CTransactionRef tx, MemPoolRemovalReason reason); public: /** Register a CScheduler to give callbacks which should run in the background (may only be called once) */ @@ -82,9 +131,14 @@ public: /** Call any remaining callbacks on the calling thread */ void FlushBackgroundCallbacks(); + /** Register with mempool to call TransactionRemovedFromMempool callbacks */ + void RegisterWithMempoolSignals(CTxMemPool& pool); + /** Unregister with mempool */ + void UnregisterWithMempoolSignals(CTxMemPool& pool); + void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); - void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &); + void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); void BlockDisconnected(const std::shared_ptr<const CBlock> &); void SetBestChain(const CBlockLocator &); void Inventory(const uint256 &); diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 64ae939672..3bd00ccb23 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "versionbits.h" -#include "consensus/params.h" +#include <versionbits.h> +#include <consensus/params.h> const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { { @@ -27,6 +27,11 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* int64_t nTimeStart = BeginTime(params); int64_t nTimeTimeout = EndTime(params); + // Check if this deployment is always active. + if (nTimeStart == Consensus::BIP9Deployment::ALWAYS_ACTIVE) { + return THRESHOLD_ACTIVE; + } + // A block's state is always the same as that of the first of its period, so it is computed based on a pindexPrev whose height equals a multiple of nPeriod - 1. if (pindexPrev != nullptr) { pindexPrev = pindexPrev->GetAncestor(pindexPrev->nHeight - ((pindexPrev->nHeight + 1) % nPeriod)); @@ -136,6 +141,11 @@ BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockI int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { + int64_t start_time = BeginTime(params); + if (start_time == Consensus::BIP9Deployment::ALWAYS_ACTIVE) { + return 0; + } + const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); // BIP 9 about state DEFINED: "The genesis block is by definition in this state for each deployment." diff --git a/src/versionbits.h b/src/versionbits.h index 71dc0c8500..0e111ed7a1 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CONSENSUS_VERSIONBITS #define BITCOIN_CONSENSUS_VERSIONBITS -#include "chain.h" +#include <chain.h> #include <map> /** What block version to use for new blocks (pre versionbits) */ diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index fc0e7c519e..15fd105779 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H -#include "policy/feerate.h" -#include "policy/fees.h" -#include "primitives/transaction.h" -#include "wallet/wallet.h" +#include <policy/feerate.h> +#include <policy/fees.h> +#include <primitives/transaction.h> +#include <wallet/wallet.h> #include <boost/optional.hpp> diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 8db3bfd69c..4cd7db048b 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypter.h" +#include <wallet/crypter.h> -#include "crypto/aes.h" -#include "crypto/sha512.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" +#include <crypto/aes.h> +#include <crypto/sha512.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> #include <string> #include <vector> @@ -152,6 +152,15 @@ bool CCryptoKeyStore::SetCrypted() return true; } +bool CCryptoKeyStore::IsLocked() const +{ + if (!IsCrypted()) { + return false; + } + LOCK(cs_KeyStore); + return vMasterKey.empty(); +} + bool CCryptoKeyStore::Lock() { if (!SetCrypted()) @@ -206,21 +215,23 @@ bool CCryptoKeyStore::Unlock(const CKeyingMaterial& vMasterKeyIn) bool CCryptoKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::AddKeyPubKey(key, pubkey); + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::AddKeyPubKey(key, pubkey); + } - if (IsLocked()) - return false; + if (IsLocked()) { + return false; + } - std::vector<unsigned char> vchCryptedSecret; - CKeyingMaterial vchSecret(key.begin(), key.end()); - if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) - return false; + std::vector<unsigned char> vchCryptedSecret; + CKeyingMaterial vchSecret(key.begin(), key.end()); + if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) { + return false; + } - if (!AddCryptedKey(pubkey, vchCryptedSecret)) - return false; + if (!AddCryptedKey(pubkey, vchCryptedSecret)) { + return false; } return true; } @@ -228,72 +239,88 @@ bool CCryptoKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) bool CCryptoKeyStore::AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { - { - LOCK(cs_KeyStore); - if (!SetCrypted()) - return false; - - mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); + LOCK(cs_KeyStore); + if (!SetCrypted()) { + return false; } + + mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); return true; } +bool CCryptoKeyStore::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::HaveKey(address); + } + return mapCryptedKeys.count(address) > 0; +} + bool CCryptoKeyStore::GetKey(const CKeyID &address, CKey& keyOut) const { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::GetKey(address, keyOut); + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::GetKey(address, keyOut); + } - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); - } + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); } return false; } bool CCryptoKeyStore::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const { + LOCK(cs_KeyStore); + if (!IsCrypted()) + return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); + vchPubKeyOut = (*mi).second.first; + return true; + } + // Check for watch-only pubkeys + return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); +} - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - vchPubKeyOut = (*mi).second.first; - return true; - } - // Check for watch-only pubkeys - return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); +std::set<CKeyID> CCryptoKeyStore::GetKeys() const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::GetKeys(); } + std::set<CKeyID> set_address; + for (const auto& mi : mapCryptedKeys) { + set_address.insert(mi.first); + } + return set_address; } bool CCryptoKeyStore::EncryptKeys(CKeyingMaterial& vMasterKeyIn) { + LOCK(cs_KeyStore); + if (!mapCryptedKeys.empty() || IsCrypted()) + return false; + + fUseCrypto = true; + for (KeyMap::value_type& mKey : mapKeys) { - LOCK(cs_KeyStore); - if (!mapCryptedKeys.empty() || IsCrypted()) + const CKey &key = mKey.second; + CPubKey vchPubKey = key.GetPubKey(); + CKeyingMaterial vchSecret(key.begin(), key.end()); + std::vector<unsigned char> vchCryptedSecret; + if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) + return false; + if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - - fUseCrypto = true; - for (KeyMap::value_type& mKey : mapKeys) - { - const CKey &key = mKey.second; - CPubKey vchPubKey = key.GetPubKey(); - CKeyingMaterial vchSecret(key.begin(), key.end()); - std::vector<unsigned char> vchCryptedSecret; - if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) - return false; - if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) - return false; - } - mapKeys.clear(); } + mapKeys.clear(); return true; } diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 1416ae7d02..7b0936ba0d 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_WALLET_CRYPTER_H #define BITCOIN_WALLET_CRYPTER_H -#include "keystore.h" -#include "serialize.h" -#include "support/allocators/secure.h" +#include <keystore.h> +#include <serialize.h> +#include <support/allocators/secure.h> #include <atomic> @@ -139,52 +139,16 @@ public: { } - bool IsCrypted() const - { - return fUseCrypto; - } - - bool IsLocked() const - { - if (!IsCrypted()) - return false; - bool result; - { - LOCK(cs_KeyStore); - result = vMasterKey.empty(); - } - return result; - } - + bool IsCrypted() const { return fUseCrypto; } + bool IsLocked() const; bool Lock(); virtual bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; - bool HaveKey(const CKeyID &address) const override - { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return CBasicKeyStore::HaveKey(address); - } - return mapCryptedKeys.count(address) > 0; - } - return false; - } + bool HaveKey(const CKeyID &address) const override; bool GetKey(const CKeyID &address, CKey& keyOut) const override; bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - std::set<CKeyID> GetKeys() const override - { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return CBasicKeyStore::GetKeys(); - } - std::set<CKeyID> set_address; - for (const auto& mi : mapCryptedKeys) { - set_address.insert(mi.first); - } - return set_address; - } + std::set<CKeyID> GetKeys() const override; /** * Wallet status (encrypted, locked) changed. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index d66ba48421..d4cd30dfac 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -3,14 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "db.h" +#include <wallet/db.h> -#include "addrman.h" -#include "fs.h" -#include "hash.h" -#include "protocol.h" -#include "util.h" -#include "utilstrencodings.h" +#include <addrman.h> +#include <hash.h> +#include <protocol.h> +#include <util.h> +#include <utilstrencodings.h> +#include <wallet/walletutil.h> #include <stdint.h> @@ -20,6 +20,40 @@ #include <boost/thread.hpp> +namespace { +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const CDBEnv& env, const std::string& filename, Db& db) +{ + if (env.IsMock()) return; + + u_int8_t fileid[DB_FILE_ID_LEN]; + int ret = db.get_mpf()->get_fileid(fileid); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.mapDb) { + u_int8_t item_fileid[DB_FILE_ID_LEN]; + if (item.second && item.second->get_mpf()->get_fileid(item_fileid) == 0 && + memcmp(fileid, item_fileid, sizeof(fileid)) == 0) { + const char* item_filename = nullptr; + item.second->get_dbname(&item_filename, nullptr); + throw std::runtime_error(strprintf("CDB: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(std::begin(item_fileid), std::end(item_fileid)), + item_filename ? item_filename : "(unknown database)")); + } + } +} +} // namespace + // // CDB // @@ -41,13 +75,12 @@ void CDBEnv::EnvShutdown() void CDBEnv::Reset() { - delete dbenv; - dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS); + dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } -CDBEnv::CDBEnv() : dbenv(nullptr) +CDBEnv::CDBEnv() { Reset(); } @@ -55,8 +88,6 @@ CDBEnv::CDBEnv() : dbenv(nullptr) CDBEnv::~CDBEnv() { EnvShutdown(); - delete dbenv; - dbenv = nullptr; } void CDBEnv::Close() @@ -148,7 +179,7 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); - Db db(dbenv, 0); + Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result == 0) return VERIFY_OK; @@ -191,7 +222,7 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco } LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0)); + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(bitdb.dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer filename.c_str(), // Filename "main", // Logical db name @@ -226,7 +257,7 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco return fSuccess; } -bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); LogPrintf("Using wallet %s\n", walletFile); @@ -234,15 +265,15 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD // Wallet file must be a plain filename without a directory if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { - errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + errorStr = strprintf(_("Wallet %s resides outside wallet directory %s"), walletFile, walletDir.string()); return false; } - if (!bitdb.Open(dataDir)) + if (!bitdb.Open(walletDir)) { // try moving the database env out of the way - fs::path pathDatabase = dataDir / "database"; - fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + fs::path pathDatabase = walletDir / "database"; + fs::path pathDatabaseBak = walletDir / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathDatabase, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); @@ -251,18 +282,18 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD } // try again - if (!bitdb.Open(dataDir)) { + if (!bitdb.Open(walletDir)) { // if it still fails, it probably means we can't even create the database env - errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); return false; } } return true; } -bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) { - if (fs::exists(dataDir / walletFile)) + if (fs::exists(walletDir / walletFile)) { std::string backup_filename; CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename); @@ -272,7 +303,7 @@ bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& data " Original %s saved as %s in %s; if" " your balance or transactions are incorrect you should" " restore from a backup."), - walletFile, backup_filename, dataDir); + walletFile, backup_filename, walletDir); } if (r == CDBEnv::RECOVER_FAIL) { @@ -300,7 +331,7 @@ bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<C std::stringstream strDump; - Db db(dbenv, 0); + Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); if (result == DB_VERIFY_BAD) { LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n"); @@ -376,38 +407,37 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb { LOCK(env->cs_db); - if (!env->Open(GetDataDir())) + if (!env->Open(GetWalletDir())) throw std::runtime_error("CDB: Failed to open database environment."); - strFile = strFilename; - ++env->mapFileUseCount[strFile]; - pdb = env->mapDb[strFile]; + pdb = env->mapDb[strFilename]; if (pdb == nullptr) { int ret; - pdb = new Db(env->dbenv, 0); + std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { - DbMpoolFile* mpf = pdb->get_mpf(); + DbMpoolFile* mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) - throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFilename)); + } } - ret = pdb->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFile.c_str(), // Filename - fMockDb ? strFile.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFilename.c_str(), // Filename + fMockDb ? strFilename.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags 0); if (ret != 0) { - delete pdb; - pdb = nullptr; - --env->mapFileUseCount[strFile]; - strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } + CheckUniqueFileid(*env, strFilename, *pdb_temp); + + pdb = pdb_temp.release(); + env->mapDb[strFilename] = pdb; if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; @@ -415,9 +445,9 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } - - env->mapDb[strFile] = pdb; } + ++env->mapFileUseCount[strFilename]; + strFile = strFilename; } } @@ -492,7 +522,7 @@ bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} CDB db(dbw, "r"); - Db* pdbCopy = new Db(env->dbenv, 0); + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename @@ -541,13 +571,12 @@ bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) } else { pdbCopy->close(0); } - delete pdbCopy; } if (fSuccess) { - Db dbA(env->dbenv, 0); + Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) fSuccess = false; - Db dbB(env->dbenv, 0); + Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) fSuccess = false; } @@ -666,12 +695,17 @@ bool CWalletDBWrapper::Backup(const std::string& strDest) env->mapFileUseCount.erase(strFile); // Copy wallet file - fs::path pathSrc = GetDataDir() / strFile; + fs::path pathSrc = GetWalletDir() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) pathDest /= strFile; try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; diff --git a/src/wallet/db.h b/src/wallet/db.h index 14283ac8f8..ed2ee65cac 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -6,12 +6,12 @@ #ifndef BITCOIN_WALLET_DB_H #define BITCOIN_WALLET_DB_H -#include "clientversion.h" -#include "fs.h" -#include "serialize.h" -#include "streams.h" -#include "sync.h" -#include "version.h" +#include <clientversion.h> +#include <fs.h> +#include <serialize.h> +#include <streams.h> +#include <sync.h> +#include <version.h> #include <atomic> #include <map> @@ -36,7 +36,7 @@ private: public: mutable CCriticalSection cs_db; - DbEnv *dbenv; + std::unique_ptr<DbEnv> dbenv; std::map<std::string, int> mapFileUseCount; std::map<std::string, Db*> mapDb; @@ -167,9 +167,9 @@ public: ideal to be called periodically */ static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); public: template <typename K, typename T> diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6abd060714..9bfcab54a5 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,19 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/validation.h" -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/fees.h" -#include "wallet/wallet.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "validation.h" //for mempool access -#include "txmempool.h" -#include "utilmoneystr.h" -#include "util.h" -#include "net.h" +#include <consensus/validation.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/fees.h> +#include <wallet/wallet.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <validation.h> //for mempool access +#include <txmempool.h> +#include <utilmoneystr.h> +#include <util.h> +#include <net.h> // Calculate the size of the transaction assuming all signatures are max size // Use DummySignatureCreator, which inserts 72 byte signatures everywhere. @@ -23,7 +23,7 @@ // calculation, but we should be able to refactor after priority is removed). // NOTE: this requires that all inputs must be in mapWallet (eg the tx should // be IsAllFromMe). -int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) +static int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet) { CMutableTransaction txNew(tx); std::vector<CInputCoin> vCoins; @@ -31,11 +31,11 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our // wallet, with a valid index into the vout array. for (auto& input : tx.vin) { - const auto mi = pWallet->mapWallet.find(input.prevout.hash); - assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + const auto mi = wallet->mapWallet.find(input.prevout.hash); + assert(mi != wallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); } - if (!pWallet->DummySignTx(txNew, vCoins)) { + if (!wallet->DummySignTx(txNew, vCoins)) { // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) // implies that we can sign for every input. return -1; @@ -43,103 +43,102 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal return GetVirtualTransactionSize(txNew); } -bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { - if (pWallet->HasWalletSpend(wtx.GetHash())) { - vErrors.push_back("Transaction has descendants in the wallet"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return false; +//! Check whether transaction has descendant in wallet or mempool, or has been +//! mined, or conflicts with a mined transaction. Return a feebumper::Result. +static feebumper::Result PreconditionChecks(const CWallet* wallet, const CWalletTx& wtx, std::vector<std::string>& errors) +{ + if (wallet->HasWalletSpend(wtx.GetHash())) { + errors.push_back("Transaction has descendants in the wallet"); + return feebumper::Result::INVALID_PARAMETER; } { LOCK(mempool.cs); auto it_mp = mempool.mapTx.find(wtx.GetHash()); if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { - vErrors.push_back("Transaction has descendants in the mempool"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return false; + errors.push_back("Transaction has descendants in the mempool"); + return feebumper::Result::INVALID_PARAMETER; } } if (wtx.GetDepthInMainChain() != 0) { - vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); - currentResult = BumpFeeResult::WALLET_ERROR; - return false; + errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + return feebumper::Result::WALLET_ERROR; } - return true; + return feebumper::Result::OK; } -CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoinControl& coin_control, CAmount totalFee) - : - txid(std::move(txidIn)), - nOldFee(0), - nNewFee(0) +namespace feebumper { + +bool TransactionCanBeBumped(CWallet* wallet, const uint256& txid) { - vErrors.clear(); - bumpedTxid.SetNull(); - AssertLockHeld(pWallet->cs_wallet); - auto it = pWallet->mapWallet.find(txid); - if (it == pWallet->mapWallet.end()) { - vErrors.push_back("Invalid or non-wallet transaction id"); - currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; - return; + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx* wtx = wallet->GetWalletTx(txid); + return wtx && SignalsOptInRBF(*wtx->tx) && !wtx->mapValue.count("replaced_by_txid"); +} + +Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, CAmount total_fee, std::vector<std::string>& errors, + CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) +{ + LOCK2(cs_main, wallet->cs_wallet); + errors.clear(); + auto it = wallet->mapWallet.find(txid); + if (it == wallet->mapWallet.end()) { + errors.push_back("Invalid or non-wallet transaction id"); + return Result::INVALID_ADDRESS_OR_KEY; } const CWalletTx& wtx = it->second; - if (!preconditionChecks(pWallet, wtx)) { - return; + Result result = PreconditionChecks(wallet, wtx, errors); + if (result != Result::OK) { + return result; } - if (!SignalsOptInRBF(wtx)) { - vErrors.push_back("Transaction is not BIP 125 replaceable"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (!SignalsOptInRBF(*wtx.tx)) { + errors.push_back("Transaction is not BIP 125 replaceable"); + return Result::WALLET_ERROR; } if (wtx.mapValue.count("replaced_by_txid")) { - vErrors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); + return Result::WALLET_ERROR; } // check that original tx consists entirely of our inputs // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!pWallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { - vErrors.push_back("Transaction contains inputs that don't belong to this wallet"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (!wallet->IsAllFromMe(*wtx.tx, ISMINE_SPENDABLE)) { + errors.push_back("Transaction contains inputs that don't belong to this wallet"); + return Result::WALLET_ERROR; } // figure out which output was change // if there was no change output or multiple change outputs, fail int nOutput = -1; for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { - if (pWallet->IsChange(wtx.tx->vout[i])) { + if (wallet->IsChange(wtx.tx->vout[i])) { if (nOutput != -1) { - vErrors.push_back("Transaction has multiple change outputs"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Transaction has multiple change outputs"); + return Result::WALLET_ERROR; } nOutput = i; } } if (nOutput == -1) { - vErrors.push_back("Transaction does not have a change output"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Transaction does not have a change output"); + return Result::WALLET_ERROR; } // Calculate the expected size of the new transaction. int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, pWallet); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, wallet); if (maxNewTxSize < 0) { - vErrors.push_back("Transaction contains inputs that cannot be signed"); - currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; - return; + errors.push_back("Transaction contains inputs that cannot be signed"); + return Result::INVALID_ADDRESS_OR_KEY; } // calculate the old fee and fee-rate - nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); - CFeeRate nOldFeeRate(nOldFee, txSize); + old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + CFeeRate nOldFeeRate(old_fee, txSize); CFeeRate nNewFeeRate; // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay @@ -149,26 +148,24 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin walletIncrementalRelayFee = ::incrementalRelayFee; } - if (totalFee > 0) { + if (total_fee > 0) { CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (totalFee < minTotalFee) { - vErrors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + if (total_fee < minTotalFee) { + errors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; + return Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(maxNewTxSize); - if (totalFee < requiredFee) { - vErrors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + if (total_fee < requiredFee) { + errors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", FormatMoney(requiredFee))); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; + return Result::INVALID_PARAMETER; } - nNewFee = totalFee; - nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); + new_fee = total_fee; + nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); } else { - nNewFee = GetMinimumFee(maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); - nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); + new_fee = GetMinimumFee(maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); + nNewFeeRate = CFeeRate(new_fee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized @@ -177,47 +174,50 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin // add 1 satoshi to the result, because it may have been rounded down. if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - nNewFee = nNewFeeRate.GetFee(maxNewTxSize); + new_fee = nNewFeeRate.GetFee(maxNewTxSize); } } // Check that in all cases the new fee doesn't violate maxTxFee - if (nNewFee > maxTxFee) { - vErrors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(nNewFee), FormatMoney(maxTxFee))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (new_fee > maxTxFee) { + errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + FormatMoney(new_fee), FormatMoney(maxTxFee))); + return Result::WALLET_ERROR; } // check that fee rate is higher than mempool's minimum fee // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a - // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. + // moment earlier. In this case, we report an error to the user, who may use total_fee to make an adjustment. CFeeRate minMempoolFeeRate = mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - vErrors.push_back(strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back(strprintf( + "New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- " + "the totalFee value should be at least %s or the settxfee value should be at least %s to add transaction", + FormatMoney(nNewFeeRate.GetFeePerK()), + FormatMoney(minMempoolFeeRate.GetFeePerK()), + FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), + FormatMoney(minMempoolFeeRate.GetFeePerK()))); + return Result::WALLET_ERROR; } // Now modify the output to increase the fee. // If the output is not large enough to pay the fee, fail. - CAmount nDelta = nNewFee - nOldFee; + CAmount nDelta = new_fee - old_fee; assert(nDelta > 0); mtx = *wtx.tx; CTxOut* poutput = &(mtx.vout[nOutput]); if (poutput->nValue < nDelta) { - vErrors.push_back("Change output is too small to bump the fee"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Change output is too small to bump the fee"); + return Result::WALLET_ERROR; } // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); - nNewFee += poutput->nValue; + new_fee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); } @@ -228,36 +228,36 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin } } - currentResult = BumpFeeResult::OK; + return Result::OK; } -bool CFeeBumper::signTransaction(CWallet *pWallet) -{ - return pWallet->SignTransaction(mtx); +bool SignTransaction(CWallet* wallet, CMutableTransaction& mtx) { + LOCK2(cs_main, wallet->cs_wallet); + return wallet->SignTransaction(mtx); } -bool CFeeBumper::commit(CWallet *pWallet) +Result CommitTransaction(CWallet* wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) { - AssertLockHeld(pWallet->cs_wallet); - if (!vErrors.empty() || currentResult != BumpFeeResult::OK) { - return false; + LOCK2(cs_main, wallet->cs_wallet); + if (!errors.empty()) { + return Result::MISC_ERROR; } - auto it = txid.IsNull() ? pWallet->mapWallet.end() : pWallet->mapWallet.find(txid); - if (it == pWallet->mapWallet.end()) { - vErrors.push_back("Invalid or non-wallet transaction id"); - currentResult = BumpFeeResult::MISC_ERROR; - return false; + auto it = txid.IsNull() ? wallet->mapWallet.end() : wallet->mapWallet.find(txid); + if (it == wallet->mapWallet.end()) { + errors.push_back("Invalid or non-wallet transaction id"); + return Result::MISC_ERROR; } CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime - if (!preconditionChecks(pWallet, oldWtx)) { - return false; + Result result = PreconditionChecks(wallet, oldWtx, errors); + if (result != Result::OK) { + return result; } - CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); + CWalletTx wtxBumped(wallet, MakeTransactionRef(std::move(mtx))); // commit/broadcast the tx - CReserveKey reservekey(pWallet); + CReserveKey reservekey(wallet); wtxBumped.mapValue = oldWtx.mapValue; wtxBumped.mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); wtxBumped.vOrderForm = oldWtx.vOrderForm; @@ -265,27 +265,29 @@ bool CFeeBumper::commit(CWallet *pWallet) wtxBumped.fTimeReceivedIsTxTime = true; wtxBumped.fFromMe = true; CValidationState state; - if (!pWallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + if (!wallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { // NOTE: CommitTransaction never returns false, so this should never happen. - vErrors.push_back(strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); - return false; + errors.push_back(strprintf("The transaction was rejected: %s", state.GetRejectReason())); + return Result::WALLET_ERROR; } - bumpedTxid = wtxBumped.GetHash(); + bumped_txid = wtxBumped.GetHash(); if (state.IsInvalid()) { // This can happen if the mempool rejected the transaction. Report // what happened in the "errors" response. - vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + errors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); } // mark the original tx as bumped - if (!pWallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { + if (!wallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { // TODO: see if JSON-RPC has a standard way of returning a response // along with an exception. It would be good to return information about // wtxBumped to the caller even if marking the original transaction // replaced does not succeed for some reason. - vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + errors.push_back("Created new bumpfee transaction but could not mark the original transaction as replaced"); } - return true; + return Result::OK; } +} // namespace feebumper + diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 3d64e53c15..8eec30440c 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -13,7 +13,9 @@ class uint256; class CCoinControl; enum class FeeEstimateMode; -enum class BumpFeeResult +namespace feebumper { + +enum class Result { OK, INVALID_ADDRESS_OR_KEY, @@ -23,39 +25,34 @@ enum class BumpFeeResult MISC_ERROR, }; -class CFeeBumper -{ -public: - CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, const CCoinControl& coin_control, CAmount totalFee); - BumpFeeResult getResult() const { return currentResult; } - const std::vector<std::string>& getErrors() const { return vErrors; } - CAmount getOldFee() const { return nOldFee; } - CAmount getNewFee() const { return nNewFee; } - uint256 getBumpedTxId() const { return bumpedTxid; } - - /* signs the new transaction, - * returns false if the tx couldn't be found or if it was - * impossible to create the signature(s) - */ - bool signTransaction(CWallet *pWallet); - - /* commits the fee bump, - * returns true, in case of CWallet::CommitTransaction was successful - * but, eventually sets vErrors if the tx could not be added to the mempool (will try later) - * or if the old transaction could not be marked as replaced - */ - bool commit(CWallet *pWalletNonConst); - -private: - bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); - - const uint256 txid; - uint256 bumpedTxid; - CMutableTransaction mtx; - std::vector<std::string> vErrors; - BumpFeeResult currentResult; - CAmount nOldFee; - CAmount nNewFee; -}; +//! Return whether transaction can be bumped. +bool TransactionCanBeBumped(CWallet* wallet, const uint256& txid); + +//! Create bumpfee transaction. +Result CreateTransaction(const CWallet* wallet, + const uint256& txid, + const CCoinControl& coin_control, + CAmount total_fee, + std::vector<std::string>& errors, + CAmount& old_fee, + CAmount& new_fee, + CMutableTransaction& mtx); + +//! Sign the new transaction, +//! @return false if the tx couldn't be found or if it was +//! impossible to create the signature(s) +bool SignTransaction(CWallet* wallet, CMutableTransaction& mtx); + +//! Commit the bumpfee transaction. +//! @return success in case of CWallet::CommitTransaction was successful, +//! but sets errors if the tx could not be added to the mempool (will try later) +//! or if the old transaction could not be marked as replaced. +Result CommitTransaction(CWallet* wallet, + const uint256& txid, + CMutableTransaction&& mtx, + std::vector<std::string>& errors, + uint256& bumped_txid); + +} // namespace feebumper #endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 76eeeeda05..73985dcf25 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -3,14 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/fees.h" +#include <wallet/fees.h> -#include "policy/policy.h" -#include "txmempool.h" -#include "util.h" -#include "validation.h" -#include "wallet/coincontrol.h" -#include "wallet/wallet.h" +#include <policy/policy.h> +#include <txmempool.h> +#include <util.h> +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/wallet.h> CAmount GetRequiredFee(unsigned int nTxBytes) diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 7b8a7dc868..225aff08ad 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H -#include "amount.h" +#include <amount.h> class CBlockPolicyEstimator; class CCoinControl; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index c984df1df8..71e7111b1a 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -3,14 +3,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/init.h" +#include <wallet/init.h> -#include "net.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validation.h" -#include "wallet/wallet.h" -#include "wallet/rpcwallet.h" +#include <net.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validation.h> +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> +#include <wallet/walletutil.h> std::string GetWalletHelpString(bool showDebug) { @@ -30,10 +31,11 @@ std::string GetWalletHelpString(bool showDebug) strUsage += HelpMessageOpt("-salvagewallet", _("Attempt to recover private keys from a corrupt wallet on startup")); strUsage += HelpMessageOpt("-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE)); strUsage += HelpMessageOpt("-txconfirmtarget=<n>", strprintf(_("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET)); - strUsage += HelpMessageOpt("-walletrbf", strprintf(_("Send transactions with full-RBF opt-in enabled (default: %u)"), DEFAULT_WALLET_RBF)); + strUsage += HelpMessageOpt("-walletrbf", strprintf(_("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)"), DEFAULT_WALLET_RBF)); strUsage += HelpMessageOpt("-upgradewallet", _("Upgrade wallet to latest format on startup")); strUsage += HelpMessageOpt("-wallet=<file>", _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), DEFAULT_WALLET_DAT)); strUsage += HelpMessageOpt("-walletbroadcast", _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST)); + strUsage += HelpMessageOpt("-walletdir=<dir>", _("Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)")); strUsage += HelpMessageOpt("-walletnotify=<cmd>", _("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)")); strUsage += HelpMessageOpt("-zapwallettxes=<mode>", _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)")); @@ -53,11 +55,16 @@ std::string GetWalletHelpString(bool showDebug) bool WalletParameterInteraction() { - gArgs.SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + for (const std::string& wallet : gArgs.GetArgs("-wallet")) { + LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet); + } - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; + } + + gArgs.SoftSetArg("-wallet", DEFAULT_WALLET_DAT); + const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && gArgs.SoftSetBoolArg("-walletbroadcast", false)) { LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); @@ -173,15 +180,27 @@ bool WalletParameterInteraction() void RegisterWalletRPC(CRPCTable &t) { - if (gArgs.GetBoolArg("-disablewallet", false)) return; + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + return; + } RegisterWalletRPCCommands(t); } bool VerifyWallets() { - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { return true; + } + + if (gArgs.IsArgSet("-walletdir") && !fs::is_directory(GetWalletDir())) { + if (fs::exists(fs::system_complete(gArgs.GetArg("-walletdir", "")))) { + return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), gArgs.GetArg("-walletdir", "").c_str())); + } + return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), gArgs.GetArg("-walletdir", "").c_str())); + } + + LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); uiInterface.InitMessage(_("Verifying wallet(s)...")); @@ -197,7 +216,7 @@ bool VerifyWallets() return InitError(strprintf(_("Error loading wallet %s. Invalid characters in -wallet filename."), walletFile)); } - fs::path wallet_path = fs::absolute(walletFile, GetDataDir()); + fs::path wallet_path = fs::absolute(walletFile, GetWalletDir()); if (fs::exists(wallet_path) && (!fs::is_regular_file(wallet_path) || fs::is_symlink(wallet_path))) { return InitError(strprintf(_("Error loading wallet %s. -wallet filename must be a regular file."), walletFile)); @@ -208,7 +227,7 @@ bool VerifyWallets() } std::string strError; - if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) { + if (!CWalletDB::VerifyEnvironment(walletFile, GetWalletDir().string(), strError)) { return InitError(strError); } @@ -222,7 +241,7 @@ bool VerifyWallets() } std::string strWarning; - bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetWalletDir().string(), strWarning, strError); if (!strWarning.empty()) { InitWarning(strWarning); } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index d6ea2a9db7..41179ebd4a 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -2,22 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "init.h" -#include "validation.h" -#include "script/script.h" -#include "script/standard.h" -#include "sync.h" -#include "util.h" -#include "utiltime.h" -#include "wallet.h" -#include "merkleblock.h" -#include "core_io.h" - -#include "rpcwallet.h" +#include <base58.h> +#include <chain.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <wallet/init.h> +#include <validation.h> +#include <script/script.h> +#include <script/standard.h> +#include <sync.h> +#include <util.h> +#include <utiltime.h> +#include <wallet/wallet.h> +#include <merkleblock.h> +#include <core_io.h> + +#include <wallet/rpcwallet.h> #include <fstream> #include <stdint.h> @@ -81,7 +81,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( "importprivkey \"privkey\" ( \"label\" ) ( rescan )\n" - "\nAdds a private key (as returned by dumpprivkey) to your wallet.\n" + "\nAdds a private key (as returned by dumpprivkey) to your wallet. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"privkey\" (string, required) The private key (see dumpprivkey)\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -165,7 +165,7 @@ UniValue abortrescan(const JSONRPCRequest& request) if (request.fHelp || request.params.size() > 0) throw std::runtime_error( "abortrescan\n" - "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nStops current wallet rescan triggered by an RPC call, e.g. by an importprivkey call.\n" "\nExamples:\n" "\nImport a private key\n" + HelpExampleCli("importprivkey", "\"mykey\"") + @@ -226,7 +226,7 @@ UniValue importaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) throw std::runtime_error( "importaddress \"address\" ( \"label\" rescan p2sh )\n" - "\nAdds a script (in hex) or address that can be watched as if it were in your wallet but cannot be used to spend.\n" + "\nAdds a script (in hex) or address that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"script\" (string, required) The hex-encoded script (or address)\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -340,7 +340,7 @@ UniValue importprunedfunds(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); - if (pwallet->IsMine(wtx)) { + if (pwallet->IsMine(*wtx.tx)) { pwallet->AddToWallet(wtx, false); return NullUniValue; } @@ -396,7 +396,7 @@ UniValue importpubkey(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) throw std::runtime_error( "importpubkey \"pubkey\" ( \"label\" rescan )\n" - "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend.\n" + "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"pubkey\" (string, required) The hex-encoded public key\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -456,7 +456,7 @@ UniValue importwallet(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "importwallet \"filename\"\n" - "\nImports keys from a wallet dump file (see dumpwallet).\n" + "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" "\nArguments:\n" "1. \"filename\" (string, required) The wallet file\n" "\nExamples:\n" @@ -500,40 +500,57 @@ UniValue importwallet(const JSONRPCRequest& request) if (vstr.size() < 2) continue; CBitcoinSecret vchSecret; - if (!vchSecret.SetString(vstr[0])) - continue; - CKey key = vchSecret.GetKey(); - CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); - CKeyID keyid = pubkey.GetID(); - if (pwallet->HaveKey(keyid)) { - LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); - continue; - } - int64_t nTime = DecodeDumpTime(vstr[1]); - std::string strLabel; - bool fLabel = true; - for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { - if (boost::algorithm::starts_with(vstr[nStr], "#")) - break; - if (vstr[nStr] == "change=1") - fLabel = false; - if (vstr[nStr] == "reserve=1") - fLabel = false; - if (boost::algorithm::starts_with(vstr[nStr], "label=")) { - strLabel = DecodeDumpString(vstr[nStr].substr(6)); - fLabel = true; + if (vchSecret.SetString(vstr[0])) { + CKey key = vchSecret.GetKey(); + CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); + CKeyID keyid = pubkey.GetID(); + if (pwallet->HaveKey(keyid)) { + LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); + continue; } + int64_t nTime = DecodeDumpTime(vstr[1]); + std::string strLabel; + bool fLabel = true; + for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { + if (boost::algorithm::starts_with(vstr[nStr], "#")) + break; + if (vstr[nStr] == "change=1") + fLabel = false; + if (vstr[nStr] == "reserve=1") + fLabel = false; + if (boost::algorithm::starts_with(vstr[nStr], "label=")) { + strLabel = DecodeDumpString(vstr[nStr].substr(6)); + fLabel = true; + } + } + LogPrintf("Importing %s...\n", EncodeDestination(keyid)); + if (!pwallet->AddKeyPubKey(key, pubkey)) { + fGood = false; + continue; + } + pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; + if (fLabel) + pwallet->SetAddressBook(keyid, strLabel, "receive"); + nTimeBegin = std::min(nTimeBegin, nTime); + } else if(IsHex(vstr[0])) { + std::vector<unsigned char> vData(ParseHex(vstr[0])); + CScript script = CScript(vData.begin(), vData.end()); + if (pwallet->HaveCScript(script)) { + LogPrintf("Skipping import of %s (script already present)\n", vstr[0]); + continue; + } + if(!pwallet->AddCScript(script)) { + LogPrintf("Error importing script %s\n", vstr[0]); + fGood = false; + continue; + } + int64_t birth_time = DecodeDumpTime(vstr[1]); + if (birth_time > 0) { + pwallet->m_script_metadata[CScriptID(script)].nCreateTime = birth_time; + nTimeBegin = std::min(nTimeBegin, birth_time); + } } - LogPrintf("Importing %s...\n", EncodeDestination(keyid)); - if (!pwallet->AddKeyPubKey(key, pubkey)) { - fGood = false; - continue; - } - pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; - if (fLabel) - pwallet->SetAddressBook(keyid, strLabel, "receive"); - nTimeBegin = std::min(nTimeBegin, nTime); } file.close(); pwallet->ShowProgress("", 100); // hide progress dialog in GUI @@ -542,7 +559,7 @@ UniValue importwallet(const JSONRPCRequest& request) pwallet->MarkDirty(); if (!fGood) - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet"); + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys/scripts to wallet"); return NullUniValue; } @@ -601,6 +618,9 @@ UniValue dumpwallet(const JSONRPCRequest& request) throw std::runtime_error( "dumpwallet \"filename\"\n" "\nDumps all wallet keys in a human-readable format to a server-side file. This does not allow overwriting existing files.\n" + "Imported scripts are included in the dumpfile, but corresponding BIP173 addresses, etc. may not be added automatically by importwallet.\n" + "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" + "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n" "\nArguments:\n" "1. \"filename\" (string, required) The filename with path (either absolute or relative to bitcoind)\n" "\nResult:\n" @@ -637,6 +657,9 @@ UniValue dumpwallet(const JSONRPCRequest& request) const std::map<CKeyID, int64_t>& mapKeyPool = pwallet->GetAllReserveKeys(); pwallet->GetKeyBirthTimes(mapKeyBirth); + std::set<CScriptID> scripts = pwallet->GetCScripts(); + // TODO: include scripts in GetKeyBirthTimes() output instead of separate + // sort time/key pairs std::vector<std::pair<int64_t, CKeyID> > vKeyBirth; for (const auto& entry : mapKeyBirth) { @@ -691,6 +714,21 @@ UniValue dumpwallet(const JSONRPCRequest& request) } } file << "\n"; + for (const CScriptID &scriptid : scripts) { + CScript script; + std::string create_time = "0"; + std::string address = EncodeDestination(scriptid); + // get birth times for scripts with metadata + auto it = pwallet->m_script_metadata.find(scriptid); + if (it != pwallet->m_script_metadata.end()) { + create_time = EncodeDumpTime(it->second.nCreateTime); + } + if(pwallet->GetCScript(scriptid, script)) { + file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); + file << strprintf(" # addr=%s\n", address); + } + } + file << "\n"; file << "# End of dump\n"; file.close(); @@ -718,8 +756,8 @@ UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int6 const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); - const bool& internal = data.exists("internal") ? data["internal"].get_bool() : false; - const bool& watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; const std::string& label = data.exists("label") && !internal ? data["label"].get_str() : ""; bool isScript = scriptPubKey.getType() == UniValue::VSTR; @@ -961,7 +999,7 @@ UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int6 pwallet->SetAddressBook(vchAddress, label, "receive"); if (pwallet->HaveKey(vchAddress)) { - return false; + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } pwallet->mapKeyMetadata[vchAddress].nCreateTime = timestamp; @@ -1039,7 +1077,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (mainRequest.fHelp || mainRequest.params.size() < 1 || mainRequest.params.size() > 2) throw std::runtime_error( "importmulti \"requests\" ( \"options\" )\n\n" - "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options).\n\n" + "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options). Requires a new wallet backup.\n\n" "Arguments:\n" "1. requests (array, required) Data to be imported\n" " [ (array of json objects)\n" diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 5d98498a4b..f839a18612 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3,29 +3,30 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" -#include "base58.h" -#include "chain.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "httpserver.h" -#include "validation.h" -#include "net.h" -#include "policy/feerate.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "rpc/mining.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "script/sign.h" -#include "timedata.h" -#include "util.h" -#include "utilmoneystr.h" -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <amount.h> +#include <base58.h> +#include <chain.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <httpserver.h> +#include <validation.h> +#include <net.h> +#include <policy/feerate.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <rpc/mining.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <script/sign.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> #include <init.h> // For StartShutdown @@ -108,7 +109,7 @@ void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) std::string rbfStatus = "no"; if (confirms <= 0) { LOCK(mempool.cs); - RBFTransactionState rbfState = IsRBFOptIn(wtx, mempool); + RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); if (rbfState == RBF_TRANSACTIONSTATE_UNKNOWN) rbfStatus = "unknown"; else if (rbfState == RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125) @@ -455,6 +456,11 @@ UniValue sendtoaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -533,6 +539,11 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); @@ -645,6 +656,11 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Bitcoin address @@ -654,7 +670,7 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) } CScript scriptPubKey = GetScriptForDestination(dest); if (!IsMine(*pwallet, scriptPubKey)) { - return ValueFromAmount(0); + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); } // Minimum confirmations @@ -707,6 +723,11 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Minimum confirmations @@ -750,6 +771,8 @@ UniValue getbalance(const JSONRPCRequest& request) throw std::runtime_error( "getbalance ( \"account\" minconf include_watchonly )\n" "\nIf account is not specified, returns the server's total available balance.\n" + "The available balance is what the wallet considers currently spendable, and is\n" + "thus affected by options which limit spendability such as -spendzeroconfchange.\n" "If account is specified (DEPRECATED), returns the balance in the account.\n" "Note that the account \"\" is not the same as leaving the parameter out.\n" "The server total may be different to the balance in the default \"\" account.\n" @@ -780,6 +803,11 @@ UniValue getbalance(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const UniValue& account_value = request.params[0]; @@ -825,6 +853,11 @@ UniValue getunconfirmedbalance(const JSONRPCRequest &request) "Returns the server's total unconfirmed balance\n"); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ValueFromAmount(pwallet->GetUnconfirmedBalance()); @@ -919,6 +952,11 @@ UniValue sendfrom(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = AccountFromValue(request.params[0]); @@ -1004,6 +1042,11 @@ UniValue sendmany(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); if (pwallet->GetBroadcastTransactions() && !g_connman) { @@ -1110,7 +1153,7 @@ UniValue addmultisigaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) { std::string msg = "addmultisigaddress nrequired [\"key\",...] ( \"account\" )\n" - "\nAdd a nrequired-to-sign multisignature address to the wallet.\n" + "\nAdd a nrequired-to-sign multisignature address to the wallet. Requires a new wallet backup.\n" "Each key is a Bitcoin address or hex-encoded public key.\n" "If 'account' is specified (DEPRECATED), assign address to that account.\n" @@ -1228,7 +1271,7 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { std::string msg = "addwitnessaddress \"address\" ( p2sh )\n" - "\nAdd a witness address for a script (with pubkey or redeemscript known).\n" + "\nAdd a witness address for a script (with pubkey or redeemscript known). Requires a new wallet backup.\n" "It returns the witness script.\n" "\nArguments:\n" @@ -1398,14 +1441,14 @@ UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bool fByA if (fByAccounts) { - for (std::map<std::string, tallyitem>::iterator it = mapAccountTally.begin(); it != mapAccountTally.end(); ++it) + for (const auto& entry : mapAccountTally) { - CAmount nAmount = (*it).second.nAmount; - int nConf = (*it).second.nConf; + CAmount nAmount = entry.second.nAmount; + int nConf = entry.second.nConf; UniValue obj(UniValue::VOBJ); - if((*it).second.fIsWatchonly) + if (entry.second.fIsWatchonly) obj.push_back(Pair("involvesWatchonly", true)); - obj.push_back(Pair("account", (*it).first)); + obj.push_back(Pair("account", entry.first)); obj.push_back(Pair("amount", ValueFromAmount(nAmount))); obj.push_back(Pair("confirmations", (nConf == std::numeric_limits<int>::max() ? 0 : nConf))); ret.push_back(obj); @@ -1455,6 +1498,11 @@ UniValue listreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, false); @@ -1495,6 +1543,11 @@ UniValue listreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, true); @@ -1683,6 +1736,11 @@ UniValue listtransactions(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = "*"; @@ -1777,6 +1835,11 @@ UniValue listaccounts(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); int nMinDepth = 1; @@ -1886,6 +1949,11 @@ UniValue listsinceblock(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const CBlockIndex* pindex = nullptr; // Block index of the specified block or the common ancestor, if the block provided was in a deactivated chain. @@ -1893,19 +1961,20 @@ UniValue listsinceblock(const JSONRPCRequest& request) int target_confirms = 1; isminefilter filter = ISMINE_SPENDABLE; - if (!request.params[0].isNull()) { + if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { uint256 blockId; blockId.SetHex(request.params[0].get_str()); BlockMap::iterator it = mapBlockIndex.find(blockId); - if (it != mapBlockIndex.end()) { - paltindex = pindex = it->second; - if (chainActive[pindex->nHeight] != pindex) { - // the block being asked for is a part of a deactivated chain; - // we don't want to depend on its perceived height in the block - // chain, we want to instead use the last common ancestor - pindex = chainActive.FindFork(pindex); - } + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + paltindex = pindex = it->second; + if (chainActive[pindex->nHeight] != pindex) { + // the block being asked for is a part of a deactivated chain; + // we don't want to depend on its perceived height in the block + // chain, we want to instead use the last common ancestor + pindex = chainActive.FindFork(pindex); } } @@ -2018,6 +2087,11 @@ UniValue gettransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2050,7 +2124,7 @@ UniValue gettransaction(const JSONRPCRequest& request) ListTransactions(pwallet, wtx, "*", 0, false, details, filter); entry.push_back(Pair("details", details)); - std::string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); + std::string strHex = EncodeHexTx(*wtx.tx, RPCSerializationFlags()); entry.push_back(Pair("hex", strHex)); return entry; @@ -2080,6 +2154,11 @@ UniValue abandontransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2114,6 +2193,10 @@ UniValue backupwallet(const JSONRPCRequest& request) + HelpExampleRpc("backupwallet", "\"backup.dat\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strDest = request.params[0].get_str(); @@ -2179,7 +2262,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + if (request.fHelp || request.params.size() != 2) { throw std::runtime_error( "walletpassphrase \"passphrase\" timeout\n" "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" @@ -2243,7 +2326,7 @@ UniValue walletpassphrasechange(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + if (request.fHelp || request.params.size() != 2) { throw std::runtime_error( "walletpassphrasechange \"oldpassphrase\" \"newpassphrase\"\n" "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n" @@ -2294,7 +2377,7 @@ UniValue walletlock(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 0)) { + if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( "walletlock\n" "\nRemoves the wallet encryption key from memory, locking the wallet.\n" @@ -2334,7 +2417,7 @@ UniValue encryptwallet(const JSONRPCRequest& request) return NullUniValue; } - if (!pwallet->IsCrypted() && (request.fHelp || request.params.size() != 1)) { + if (request.fHelp || request.params.size() != 1) { throw std::runtime_error( "encryptwallet \"passphrase\"\n" "\nEncrypts the wallet with 'passphrase'. This is for first time encryption.\n" @@ -2433,6 +2516,10 @@ UniValue lockunspent(const JSONRPCRequest& request) + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); RPCTypeCheckArgument(request.params[0], UniValue::VBOOL); @@ -2447,12 +2534,15 @@ UniValue lockunspent(const JSONRPCRequest& request) RPCTypeCheckArgument(request.params[1], UniValue::VARR); - UniValue outputs = request.params[1].get_array(); - for (unsigned int idx = 0; idx < outputs.size(); idx++) { - const UniValue& output = outputs[idx]; - if (!output.isObject()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected object"); - const UniValue& o = output.get_obj(); + const UniValue& output_params = request.params[1]; + + // Create and validate the COutPoints first. + + std::vector<COutPoint> outputs; + outputs.reserve(output_params.size()); + + for (unsigned int idx = 0; idx < output_params.size(); idx++) { + const UniValue& o = output_params[idx].get_obj(); RPCTypeCheckObj(o, { @@ -2460,20 +2550,50 @@ UniValue lockunspent(const JSONRPCRequest& request) {"vout", UniValueType(UniValue::VNUM)}, }); - std::string txid = find_value(o, "txid").get_str(); - if (!IsHex(txid)) + const std::string& txid = find_value(o, "txid").get_str(); + if (!IsHex(txid)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected hex txid"); + } - int nOutput = find_value(o, "vout").get_int(); - if (nOutput < 0) + const int nOutput = find_value(o, "vout").get_int(); + if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); + } - COutPoint outpt(uint256S(txid), nOutput); + const COutPoint outpt(uint256S(txid), nOutput); - if (fUnlock) - pwallet->UnlockCoin(outpt); - else - pwallet->LockCoin(outpt); + const auto it = pwallet->mapWallet.find(outpt.hash); + if (it == pwallet->mapWallet.end()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, unknown transaction"); + } + + const CWalletTx& trans = it->second; + + if (outpt.n >= trans.tx->vout.size()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout index out of bounds"); + } + + if (pwallet->IsSpent(outpt.hash, outpt.n)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected unspent output"); + } + + const bool is_locked = pwallet->IsLockedCoin(outpt.hash, outpt.n); + + if (fUnlock && !is_locked) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected locked output"); + } + + if (!fUnlock && is_locked) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, output already locked"); + } + + outputs.push_back(outpt); + } + + // Atomically set (un)locked status for the outputs. + for (const COutPoint& outpt : outputs) { + if (fUnlock) pwallet->UnlockCoin(outpt); + else pwallet->LockCoin(outpt); } return true; @@ -2592,6 +2712,11 @@ UniValue getwalletinfo(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); @@ -2801,9 +2926,12 @@ UniValue listunspent(const JSONRPCRequest& request) nMaximumCount = options["maximumCount"].get_int64(); } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; - assert(pwallet != nullptr); LOCK2(cs_main, pwallet->cs_wallet); pwallet->AvailableCoins(vecOutputs, !include_unsafe, nullptr, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); @@ -2854,9 +2982,9 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( - "fundrawtransaction \"hexstring\" ( options )\n" + "fundrawtransaction \"hexstring\" ( options iswitness )\n" "\nAdd inputs to a transaction until it has enough in value to meet its out value.\n" "This will not modify existing inputs, and will add at most one change output to the outputs.\n" "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" @@ -2891,6 +3019,9 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) " \"CONSERVATIVE\"\n" " }\n" " for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}\n" + "3. iswitness (boolean, optional) Whether the transaction hex is a serialized witness transaction \n" + " If iswitness is not present, heuristic tests will be used in decoding\n" + "\nResult:\n" "{\n" " \"hex\": \"value\", (string) The resulting raw transaction (hex-encoded string)\n" @@ -2911,6 +3042,10 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) ObserveSafeMode(); RPCTypeCheck(request.params, {UniValue::VSTR}); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + CCoinControl coinControl; int changePosition = -1; bool lockUnspents = false; @@ -2923,7 +3058,7 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) coinControl.fAllowWatchOnly = request.params[1].get_bool(); } else { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ, UniValue::VBOOL}); UniValue options = request.params[1]; @@ -2992,8 +3127,11 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) // parse hex string from parameter CMutableTransaction tx; - if (!DecodeHexTx(tx, request.params[0].get_str(), true)) + bool try_witness = request.params[2].isNull() ? true : request.params[2].get_bool(); + bool try_no_witness = request.params[2].isNull() ? true : !request.params[2].get_bool(); + if (!DecodeHexTx(tx, request.params[0].get_str(), try_no_witness, try_witness)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + } if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); @@ -3121,48 +3259,57 @@ UniValue bumpfee(const JSONRPCRequest& request) } } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - CFeeBumper feeBump(pwallet, hash, coin_control, totalFee); - BumpFeeResult res = feeBump.getResult(); - if (res != BumpFeeResult::OK) - { + + std::vector<std::string> errors; + CAmount old_fee; + CAmount new_fee; + CMutableTransaction mtx; + feebumper::Result res = feebumper::CreateTransaction(pwallet, hash, coin_control, totalFee, errors, old_fee, new_fee, mtx); + if (res != feebumper::Result::OK) { switch(res) { - case BumpFeeResult::INVALID_ADDRESS_OR_KEY: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_ADDRESS_OR_KEY: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0]); break; - case BumpFeeResult::INVALID_REQUEST: - throw JSONRPCError(RPC_INVALID_REQUEST, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_REQUEST: + throw JSONRPCError(RPC_INVALID_REQUEST, errors[0]); break; - case BumpFeeResult::INVALID_PARAMETER: - throw JSONRPCError(RPC_INVALID_PARAMETER, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_PARAMETER: + throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0]); break; - case BumpFeeResult::WALLET_ERROR: - throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + case feebumper::Result::WALLET_ERROR: + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); break; default: - throw JSONRPCError(RPC_MISC_ERROR, feeBump.getErrors()[0]); + throw JSONRPCError(RPC_MISC_ERROR, errors[0]); break; } } // sign bumped transaction - if (!feeBump.signTransaction(pwallet)) { + if (!feebumper::SignTransaction(pwallet, mtx)) { throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } // commit the bumped transaction - if(!feeBump.commit(pwallet)) { - throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + uint256 txid; + if (feebumper::CommitTransaction(pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); } UniValue result(UniValue::VOBJ); - result.push_back(Pair("txid", feeBump.getBumpedTxId().GetHex())); - result.push_back(Pair("origfee", ValueFromAmount(feeBump.getOldFee()))); - result.push_back(Pair("fee", ValueFromAmount(feeBump.getNewFee()))); - UniValue errors(UniValue::VARR); - for (const std::string& err: feeBump.getErrors()) - errors.push_back(err); - result.push_back(Pair("errors", errors)); + result.push_back(Pair("txid", txid.GetHex())); + result.push_back(Pair("origfee", ValueFromAmount(old_fee))); + result.push_back(Pair("fee", ValueFromAmount(new_fee))); + UniValue result_errors(UniValue::VARR); + for (const std::string& error : errors) { + result_errors.push_back(error); + } + result.push_back(Pair("errors", result_errors)); return result; } @@ -3212,6 +3359,81 @@ UniValue generate(const JSONRPCRequest& request) return generateBlocks(coinbase_script, num_generate, max_tries, true); } +UniValue rescanblockchain(const JSONRPCRequest& request) +{ + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 2) { + throw std::runtime_error( + "rescanblockchain (\"start_height\") (\"stop_height\")\n" + "\nRescan the local blockchain for wallet related transactions.\n" + "\nArguments:\n" + "1. \"start_height\" (numeric, optional) block height where the rescan should start\n" + "2. \"stop_height\" (numeric, optional) the last block height that should be scanned\n" + "\nResult:\n" + "{\n" + " \"start_height\" (numeric) The block height where the rescan has started. If omitted, rescan started from the genesis block.\n" + " \"stop_height\" (numeric) The height of the last rescanned block. If omitted, rescan stopped at the chain tip.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("rescanblockchain", "100000 120000") + + HelpExampleRpc("rescanblockchain", "100000, 120000") + ); + } + + LOCK2(cs_main, pwallet->cs_wallet); + + CBlockIndex *pindexStart = chainActive.Genesis(); + CBlockIndex *pindexStop = nullptr; + if (!request.params[0].isNull()) { + pindexStart = chainActive[request.params[0].get_int()]; + if (!pindexStart) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); + } + } + + if (!request.params[1].isNull()) { + pindexStop = chainActive[request.params[1].get_int()]; + if (!pindexStop) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); + } + else if (pindexStop->nHeight < pindexStart->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater then start_height"); + } + } + + // We can't rescan beyond non-pruned blocks, stop and throw an error + if (fPruneMode) { + CBlockIndex *block = pindexStop ? pindexStop : chainActive.Tip(); + while (block && block->nHeight >= pindexStart->nHeight) { + if (!(block->nStatus & BLOCK_HAVE_DATA)) { + throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); + } + block = block->pprev; + } + } + + CBlockIndex *stopBlock = pwallet->ScanForWalletTransactions(pindexStart, pindexStop, true); + if (!stopBlock) { + if (pwallet->IsAbortingRescan()) { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted."); + } + // if we got a nullptr returned, ScanForWalletTransactions did rescan up to the requested stopindex + stopBlock = pindexStop ? pindexStop : chainActive.Tip(); + } + else { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan failed. Potentially corrupted data files."); + } + + UniValue response(UniValue::VOBJ); + response.pushKV("start_height", pindexStart->nHeight); + response.pushKV("stop_height", stopBlock->nHeight); + return response; +} + extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); @@ -3222,11 +3444,12 @@ extern UniValue importwallet(const JSONRPCRequest& request); extern UniValue importprunedfunds(const JSONRPCRequest& request); extern UniValue removeprunedfunds(const JSONRPCRequest& request); extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue rescanblockchain(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options"} }, + { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, @@ -3276,6 +3499,7 @@ static const CRPCCommand commands[] = { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, + { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, { "generating", "generate", &generate, {"nblocks","maxtries"} }, }; diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index 330878ceb5..b95bb14335 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -2,26 +2,24 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" +#include <wallet/wallet.h> -#include "wallet/test/wallet_test_fixture.h" +#include <wallet/test/wallet_test_fixture.h> #include <stdint.h> #include <boost/test/unit_test.hpp> -extern CWallet* pwalletMain; - BOOST_FIXTURE_TEST_SUITE(accounting_tests, WalletTestingSetup) static void -GetResults(std::map<CAmount, CAccountingEntry>& results) +GetResults(CWallet *wallet, std::map<CAmount, CAccountingEntry>& results) { std::list<CAccountingEntry> aes; results.clear(); - BOOST_CHECK(pwalletMain->ReorderTransactions() == DB_LOAD_OK); - pwalletMain->ListAccountCreditDebit("", aes); + BOOST_CHECK(wallet->ReorderTransactions() == DB_LOAD_OK); + wallet->ListAccountCreditDebit("", aes); for (CAccountingEntry& ae : aes) { results[ae.nOrderPos] = ae; @@ -54,7 +52,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.strOtherAccount = "c"; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(pwalletMain->nOrderPosNext == 3); BOOST_CHECK(2 == results.size()); @@ -70,7 +68,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = pwalletMain->IncOrderPosNext(); pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 4); @@ -83,7 +81,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) wtx.mapValue["comment"] = "y"; { - CMutableTransaction tx(wtx); + CMutableTransaction tx(*wtx.tx); --tx.nLockTime; // Just to change the hash :) wtx.SetTx(MakeTransactionRef(std::move(tx))); } @@ -93,7 +91,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) wtx.mapValue["comment"] = "x"; { - CMutableTransaction tx(wtx); + CMutableTransaction tx(*wtx.tx); --tx.nLockTime; // Just to change the hash :) wtx.SetTx(MakeTransactionRef(std::move(tx))); } @@ -102,7 +100,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) vpwtx[2]->nTimeReceived = (unsigned int)1333333329; vpwtx[2]->nOrderPos = -1; - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 6); @@ -120,7 +118,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = -1; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 4); BOOST_CHECK(pwalletMain->nOrderPosNext == 7); diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index f4dabc50c0..3ff8c6d224 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "test/test_bitcoin.h" -#include "utilstrencodings.h" -#include "wallet/crypter.h" +#include <test/test_bitcoin.h> +#include <utilstrencodings.h> +#include <wallet/crypter.h> #include <vector> diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index e2f48c45ab..3ee83d2d7c 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -2,13 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/test/wallet_test_fixture.h" +#include <wallet/test/wallet_test_fixture.h> -#include "rpc/server.h" -#include "wallet/db.h" -#include "wallet/wallet.h" - -CWallet *pwalletMain; +#include <rpc/server.h> +#include <wallet/db.h> WalletTestingSetup::WalletTestingSetup(const std::string& chainName): TestingSetup(chainName) @@ -17,18 +14,16 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bool fFirstRun; std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); - pwalletMain = new CWallet(std::move(dbw)); + pwalletMain = MakeUnique<CWallet>(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); - RegisterValidationInterface(pwalletMain); + RegisterValidationInterface(pwalletMain.get()); RegisterWalletRPCCommands(tableRPC); } WalletTestingSetup::~WalletTestingSetup() { - UnregisterValidationInterface(pwalletMain); - delete pwalletMain; - pwalletMain = nullptr; + UnregisterValidationInterface(pwalletMain.get()); bitdb.Flush(true); bitdb.Reset(); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 9373b7907c..292d654438 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -5,13 +5,17 @@ #ifndef BITCOIN_WALLET_TEST_FIXTURE_H #define BITCOIN_WALLET_TEST_FIXTURE_H -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> + +#include <wallet/wallet.h> /** Testing setup and teardown for wallet. */ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~WalletTestingSetup(); + + std::unique_ptr<CWallet> pwalletMain; }; #endif diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 5ebacd57d3..80e31a1ce0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -2,25 +2,23 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" +#include <wallet/wallet.h> #include <set> #include <stdint.h> #include <utility> #include <vector> -#include "consensus/validation.h" -#include "rpc/server.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "wallet/coincontrol.h" -#include "wallet/test/wallet_test_fixture.h" +#include <consensus/validation.h> +#include <rpc/server.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> #include <univalue.h> -extern CWallet* pwalletMain; - extern UniValue importmulti(const JSONRPCRequest& request); extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); @@ -386,7 +384,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip, nullptr)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -399,7 +397,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip, nullptr)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -489,6 +487,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) vpwallets[0] = &wallet; ::importwallet(request); + LOCK(wallet.cs_wallet); BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); for (size_t i = 0; i < coinbaseTxns.size(); ++i) { @@ -534,6 +533,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { + LOCK(cs_main); auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; @@ -547,6 +547,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 wtx.SetMerkleBranch(block, 0); } wallet.AddToWallet(wtx); + LOCK(wallet.cs_wallet); return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; } @@ -583,6 +584,7 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) BOOST_AUTO_TEST_CASE(LoadReceiveRequests) { CTxDestination dest = CKeyID(); + LOCK(pwalletMain->cs_wallet); pwalletMain->AddDestData(dest, "misc", "val_misc"); pwalletMain->AddDestData(dest, "rr0", "val_rr0"); pwalletMain->AddDestData(dest, "rr1", "val_rr1"); @@ -604,7 +606,7 @@ public: bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - wallet->ScanForWalletTransactions(chainActive.Genesis()); + wallet->ScanForWalletTransactions(chainActive.Genesis(), nullptr); } ~ListCoinsTestingSetup() @@ -625,6 +627,7 @@ public: BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error, dummy)); CValidationState state; BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + LOCK(wallet->cs_wallet); auto it = wallet->mapWallet.find(wtx.GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 925b474d73..dafd708d09 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3,36 +3,35 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" - -#include "base58.h" -#include "checkpoints.h" -#include "chain.h" -#include "wallet/coincontrol.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "fs.h" -#include "init.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "net.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "script/sign.h" -#include "scheduler.h" -#include "timedata.h" -#include "txmempool.h" -#include "util.h" -#include "ui_interface.h" -#include "utilmoneystr.h" -#include "wallet/fees.h" +#include <wallet/wallet.h> + +#include <base58.h> +#include <checkpoints.h> +#include <chain.h> +#include <wallet/coincontrol.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <fs.h> +#include <wallet/init.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <net.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <scheduler.h> +#include <timedata.h> +#include <txmempool.h> +#include <util.h> +#include <utilmoneystr.h> +#include <wallet/fees.h> #include <assert.h> +#include <future> #include <boost/algorithm/string/replace.hpp> #include <boost/thread.hpp> @@ -283,7 +282,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, } } -bool CWallet::LoadKeyMetadata(const CTxDestination& keyID, const CKeyMetadata &meta) +bool CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &meta) { AssertLockHeld(cs_wallet); // mapKeyMetadata UpdateTimeFirstKey(meta.nCreateTime); @@ -291,6 +290,14 @@ bool CWallet::LoadKeyMetadata(const CTxDestination& keyID, const CKeyMetadata &m return true; } +bool CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &meta) +{ + AssertLockHeld(cs_wallet); // m_script_metadata + UpdateTimeFirstKey(meta.nCreateTime); + m_script_metadata[script_id] = meta; + return true; +} + bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { return CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret); @@ -339,7 +346,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) { if (!CCryptoKeyStore::AddWatchOnly(dest)) return false; - const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; + const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); return CWalletDB(*dbw).WriteWatchOnly(dest, meta); @@ -347,7 +354,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) { - mapKeyMetadata[CScriptID(dest)].nCreateTime = nCreateTime; + m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; return AddWatchOnly(dest); } @@ -532,6 +539,9 @@ void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> ran copyFrom = &mapWallet[hash]; } } + + assert(copyFrom); + // Now copy data from copyFrom to rest: for (TxSpends::iterator it = range.first; it != range.second; ++it) { @@ -701,9 +711,9 @@ DBErrors CWallet::ReorderTransactions() typedef std::multimap<int64_t, TxPair > TxItems; TxItems txByTime; - for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (auto& entry : mapWallet) { - CWalletTx* wtx = &((*it).second); + CWalletTx* wtx = &entry.second; txByTime.insert(std::make_pair(wtx->nTimeReceived, TxPair(wtx, nullptr))); } std::list<CAccountingEntry> acentries; @@ -1214,6 +1224,19 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pin void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { LOCK2(cs_main, cs_wallet); SyncTransaction(ptx); + + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = true; + } +} + +void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { + LOCK(cs_wallet); + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = false; + } } void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { @@ -1228,10 +1251,14 @@ void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const for (const CTransactionRef& ptx : vtxConflicted) { SyncTransaction(ptx); + TransactionRemovedFromMempool(ptx); } for (size_t i = 0; i < pblock->vtx.size(); i++) { SyncTransaction(pblock->vtx[i], pindex, i); + TransactionRemovedFromMempool(pblock->vtx[i]); } + + m_last_block_processed = pindex; } void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { @@ -1244,6 +1271,36 @@ void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { +void CWallet::BlockUntilSyncedToCurrentChain() { + AssertLockNotHeld(cs_main); + AssertLockNotHeld(cs_wallet); + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip()... + // We could also take cs_wallet here, and call m_last_block_processed + // protected by cs_wallet instead of cs_main, but as long as we need + // cs_main here anyway, its easier to just call it cs_main-protected. + LOCK(cs_main); + const CBlockIndex* initialChainTip = chainActive.Tip(); + + if (m_last_block_processed->GetAncestor(initialChainTip->nHeight) == initialChainTip) { + return; + } + } + + // ...otherwise put a callback in the validation interface queue and wait + // for the queue to drain enough to execute it (indicating we are caught up + // at least with the time we entered this function). + + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); + promise.get_future().wait(); +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { { @@ -1568,7 +1625,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update) LogPrintf("%s: Rescanning last %i blocks\n", __func__, startBlock ? chainActive.Height() - startBlock->nHeight + 1 : 0); if (startBlock) { - const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, update); + const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, nullptr, update); if (failedBlock) { return failedBlock->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1; } @@ -1584,12 +1641,19 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update) * Returns null if scan was successful. Otherwise, if a complete rescan was not * possible (due to pruning or corruption), returns pointer to the most recent * block that could not be scanned. + * + * If pindexStop is not a nullptr, the scan will stop at the block-index + * defined by pindexStop */ -CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) +CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate) { int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); + if (pindexStop) { + assert(pindexStop->nHeight >= pindexStart->nHeight); + } + CBlockIndex* pindex = pindexStart; CBlockIndex* ret = nullptr; { @@ -1617,6 +1681,9 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f } else { ret = pindex; } + if (pindex == pindexStop) { + break; + } pindex = chainActive.Next(pindex); } if (pindex && fAbortRescan) { @@ -1652,11 +1719,8 @@ void CWallet::ReacceptWalletTransactions() } // Try to add wallet transactions to memory pool - for (std::pair<const int64_t, CWalletTx*>& item : mapSorted) - { + for (std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); - - LOCK(mempool.cs); CValidationState state; wtx.AcceptToMemoryPool(maxTxFee, state); } @@ -1708,7 +1772,7 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const debit += nDebitCached; else { - nDebitCached = pwallet->GetDebit(*this, ISMINE_SPENDABLE); + nDebitCached = pwallet->GetDebit(*tx, ISMINE_SPENDABLE); fDebitCached = true; debit += nDebitCached; } @@ -1719,7 +1783,7 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const debit += nWatchDebitCached; else { - nWatchDebitCached = pwallet->GetDebit(*this, ISMINE_WATCH_ONLY); + nWatchDebitCached = pwallet->GetDebit(*tx, ISMINE_WATCH_ONLY); fWatchDebitCached = true; debit += nWatchDebitCached; } @@ -1741,7 +1805,7 @@ CAmount CWalletTx::GetCredit(const isminefilter& filter) const credit += nCreditCached; else { - nCreditCached = pwallet->GetCredit(*this, ISMINE_SPENDABLE); + nCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); fCreditCached = true; credit += nCreditCached; } @@ -1752,7 +1816,7 @@ CAmount CWalletTx::GetCredit(const isminefilter& filter) const credit += nWatchCreditCached; else { - nWatchCreditCached = pwallet->GetCredit(*this, ISMINE_WATCH_ONLY); + nWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); fWatchCreditCached = true; credit += nWatchCreditCached; } @@ -1766,7 +1830,7 @@ CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const { if (fUseCache && fImmatureCreditCached) return nImmatureCreditCached; - nImmatureCreditCached = pwallet->GetCredit(*this, ISMINE_SPENDABLE); + nImmatureCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); fImmatureCreditCached = true; return nImmatureCreditCached; } @@ -1804,13 +1868,13 @@ CAmount CWalletTx::GetAvailableCredit(bool fUseCache) const return nCredit; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool& fUseCache) const +CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const { if (IsCoinBase() && GetBlocksToMaturity() > 0 && IsInMainChain()) { if (fUseCache && fImmatureWatchCreditCached) return nImmatureWatchCreditCached; - nImmatureWatchCreditCached = pwallet->GetCredit(*this, ISMINE_WATCH_ONLY); + nImmatureWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); fImmatureWatchCreditCached = true; return nImmatureWatchCreditCached; } @@ -1818,7 +1882,7 @@ CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool& fUseCache) const return 0; } -CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool& fUseCache) const +CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool fUseCache) const { if (pwallet == nullptr) return 0; @@ -1851,21 +1915,20 @@ CAmount CWalletTx::GetChange() const { if (fChangeCached) return nChangeCached; - nChangeCached = pwallet->GetChange(*this); + nChangeCached = pwallet->GetChange(*tx); fChangeCached = true; return nChangeCached; } bool CWalletTx::InMempool() const { - LOCK(mempool.cs); - return mempool.exists(GetHash()); + return fInMempool; } bool CWalletTx::IsTrusted() const { // Quick answer in most cases - if (!CheckFinalTx(*this)) + if (!CheckFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(); if (nDepth >= 1) @@ -1966,9 +2029,9 @@ CAmount CWallet::GetBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (pcoin->IsTrusted()) nTotal += pcoin->GetAvailableCredit(); } @@ -1982,9 +2045,9 @@ CAmount CWallet::GetUnconfirmedBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) nTotal += pcoin->GetAvailableCredit(); } @@ -1997,9 +2060,9 @@ CAmount CWallet::GetImmatureBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; nTotal += pcoin->GetImmatureCredit(); } } @@ -2011,9 +2074,9 @@ CAmount CWallet::GetWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (pcoin->IsTrusted()) nTotal += pcoin->GetAvailableWatchOnlyCredit(); } @@ -2027,9 +2090,9 @@ CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) nTotal += pcoin->GetAvailableWatchOnlyCredit(); } @@ -2042,9 +2105,9 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; nTotal += pcoin->GetImmatureWatchOnlyCredit(); } } @@ -2109,7 +2172,7 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const return balance; } -void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t nMaximumCount, const int nMinDepth, const int nMaxDepth) const { vCoins.clear(); @@ -2118,12 +2181,12 @@ void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CAmount nTotal = 0; - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const uint256& wtxid = it->first; - const CWalletTx* pcoin = &(*it).second; + const uint256& wtxid = entry.first; + const CWalletTx* pcoin = &entry.second; - if (!CheckFinalTx(*pcoin)) + if (!CheckFinalTx(*pcoin->tx)) continue; if (pcoin->IsCoinBase() && pcoin->GetBlocksToMaturity() > 0) @@ -2182,10 +2245,10 @@ void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) continue; - if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) continue; - if (IsLockedCoin((*it).first, i)) + if (IsLockedCoin(entry.first, i)) continue; if (IsSpent(wtxid, i)) @@ -2532,9 +2595,8 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC { std::vector<CRecipient> vecSend; - // Turn the txout set into a CRecipient vector - for (size_t idx = 0; idx < tx.vout.size(); idx++) - { + // Turn the txout set into a CRecipient vector. + for (size_t idx = 0; idx < tx.vout.size(); idx++) { const CTxOut& txOut = tx.vout[idx]; CRecipient recipient = {txOut.scriptPubKey, txOut.nValue, setSubtractFeeFromOutputs.count(idx) == 1}; vecSend.push_back(recipient); @@ -2542,8 +2604,13 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC coinControl.fAllowOtherInputs = true; - for (const CTxIn& txin : tx.vin) + for (const CTxIn& txin : tx.vin) { coinControl.Select(txin.prevout); + } + + // Acquire the locks to prevent races to the new locked unspents between the + // CreateTransaction call and LockCoin calls (when lockUnspents is true). + LOCK2(cs_main, cs_wallet); CReserveKey reservekey(this); CWalletTx wtx; @@ -2553,31 +2620,28 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC if (nChangePosInOut != -1) { tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.tx->vout[nChangePosInOut]); - // we don't have the normal Create/Commit cycle, and don't want to risk reusing change, - // so just remove the key from the keypool here. + // We don't have the normal Create/Commit cycle, and don't want to risk + // reusing change, so just remove the key from the keypool here. reservekey.KeepKey(); } - // Copy output sizes from new transaction; they may have had the fee subtracted from them - for (unsigned int idx = 0; idx < tx.vout.size(); idx++) + // Copy output sizes from new transaction; they may have had the fee + // subtracted from them. + for (unsigned int idx = 0; idx < tx.vout.size(); idx++) { tx.vout[idx].nValue = wtx.tx->vout[idx].nValue; + } - // Add new txins (keeping original txin scriptSig/order) - for (const CTxIn& txin : wtx.tx->vin) - { - if (!coinControl.IsSelected(txin.prevout)) - { + // Add new txins while keeping original txin scriptSig/order. + for (const CTxIn& txin : wtx.tx->vin) { + if (!coinControl.IsSelected(txin.prevout)) { tx.vin.push_back(txin); - if (lockUnspents) - { - LOCK2(cs_main, cs_wallet); - LockCoin(txin.prevout); + if (lockUnspents) { + LockCoin(txin.prevout); } } } - return true; } @@ -2704,6 +2768,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (recipient.fSubtractFeeFromAmount) { + assert(nSubtractFeeFromAmount != 0); txout.nValue -= nFeeRet / nSubtractFeeFromAmount; // Subtract fee equally from each selected recipient if (fFirst) // first receiver pays the remainder not divisible by output count @@ -2904,7 +2969,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT wtxNew.SetTx(MakeTransactionRef(std::move(txNew))); // Limit size - if (GetTransactionWeight(wtxNew) >= MAX_STANDARD_TX_WEIGHT) + if (GetTransactionWeight(*wtxNew.tx) >= MAX_STANDARD_TX_WEIGHT) { strFailReason = _("Transaction too large"); return false; @@ -2966,14 +3031,18 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon // Track how many getdata requests our transaction gets mapRequestCount[wtxNew.GetHash()] = 0; + // Get the inserted-CWalletTx from mapWallet so that the + // fInMempool flag is cached properly + CWalletTx& wtx = mapWallet[wtxNew.GetHash()]; + if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(maxTxFee, state)) { LogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", state.GetRejectReason()); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtxNew.RelayWalletTransaction(connman); + wtx.RelayWalletTransaction(connman); } } } @@ -3640,9 +3709,9 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c // find first block that affects those keys, if there are any left std::vector<CKeyID> vAffected; - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); it++) { + for (const auto& entry : mapWallet) { // iterate over all wallet transactions... - const CWalletTx &wtx = (*it).second; + const CWalletTx &wtx = entry.second; BlockMap::const_iterator blit = mapBlockIndex.find(wtx.hashBlock); if (blit != mapBlockIndex.end() && chainActive.Contains(blit->second)) { // ... which are already in a block @@ -3662,8 +3731,8 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c } // Extract block timestamps for those keys - for (std::map<CKeyID, CBlockIndex*>::const_iterator it = mapKeyFirstBlock.begin(); it != mapKeyFirstBlock.end(); it++) - mapKeyBirth[it->first] = it->second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off + for (const auto& entry : mapKeyFirstBlock) + mapKeyBirth[entry.first] = entry.second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off } /** @@ -3792,7 +3861,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) uiInterface.InitMessage(_("Zapping all transactions from wallet...")); std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); - std::unique_ptr<CWallet> tempWallet(new CWallet(std::move(dbw))); + std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3870,7 +3939,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Top up the keypool if (!walletInstance->TopUpKeyPool()) { InitError(_("Unable to generate initial keys") += "\n"); - return NULL; + return nullptr; } walletInstance->SetBestChain(chainActive.GetLocator()); @@ -3889,8 +3958,6 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) LogPrintf(" wallet %15dms\n", GetTimeMillis() - nStart); - RegisterValidationInterface(walletInstance); - // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); @@ -3902,6 +3969,10 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); } + + walletInstance->m_last_block_processed = chainActive.Tip(); + RegisterValidationInterface(walletInstance); + if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { //We can't rescan beyond non-pruned blocks, stop and throw an error @@ -3929,7 +4000,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) } nStart = GetTimeMillis(); - walletInstance->ScanForWalletTransactions(pindexRescan, true); + walletInstance->ScanForWalletTransactions(pindexRescan, nullptr, true); LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart); walletInstance->SetBestChain(chainActive.GetLocator()); walletInstance->dbw->IncrementUpdateCounter(); @@ -4045,8 +4116,20 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - return ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, + // Quick check to avoid re-setting fInMempool to false + if (mempool.exists(tx->GetHash())) { + return false; + } + + // We must set fInMempool here - while it will be re-set to true by the + // entered-mempool callback, if we did not there would be a race where a + // user could call sendmoney in a loop and hit spurious out of funds errors + // because we think that the transaction they just generated's change is + // unavailable as we're not yet aware its in mempool. + bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + fInMempool = ret; + return ret; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c4af192f36..97581794a8 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -6,18 +6,18 @@ #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H -#include "amount.h" -#include "policy/feerate.h" -#include "streams.h" -#include "tinyformat.h" -#include "ui_interface.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "script/ismine.h" -#include "script/sign.h" -#include "wallet/crypter.h" -#include "wallet/walletdb.h" -#include "wallet/rpcwallet.h" +#include <amount.h> +#include <policy/feerate.h> +#include <streams.h> +#include <tinyformat.h> +#include <ui_interface.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <script/ismine.h> +#include <script/sign.h> +#include <wallet/crypter.h> +#include <wallet/walletdb.h> +#include <wallet/rpcwallet.h> #include <algorithm> #include <atomic> @@ -214,10 +214,6 @@ public: Init(); } - /** Helper conversion operator to allow passing CMerkleTx where CTransaction is expected. - * TODO: adapt callers and remove this operator. */ - operator const CTransaction&() const { return *tx; } - void Init() { hashBlock = uint256(); @@ -252,8 +248,6 @@ public: int GetDepthInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet); } bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; - /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -330,6 +324,7 @@ public: mutable bool fImmatureWatchCreditCached; mutable bool fAvailableWatchCreditCached; mutable bool fChangeCached; + mutable bool fInMempool; mutable CAmount nDebitCached; mutable CAmount nCreditCached; mutable CAmount nImmatureCreditCached; @@ -369,6 +364,7 @@ public: fImmatureWatchCreditCached = false; fAvailableWatchCreditCached = false; fChangeCached = false; + fInMempool = false; nDebitCached = 0; nCreditCached = 0; nImmatureCreditCached = 0; @@ -449,8 +445,8 @@ public: CAmount GetCredit(const isminefilter& filter) const; CAmount GetImmatureCredit(bool fUseCache=true) const; CAmount GetAvailableCredit(bool fUseCache=true) const; - CAmount GetImmatureWatchOnlyCredit(const bool& fUseCache=true) const; - CAmount GetAvailableWatchOnlyCredit(const bool& fUseCache=true) const; + CAmount GetImmatureWatchOnlyCredit(const bool fUseCache=true) const; + CAmount GetAvailableWatchOnlyCredit(const bool fUseCache=true) const; CAmount GetChange() const; void GetAmounts(std::list<COutputEntry>& listReceived, @@ -473,6 +469,9 @@ public: // RelayWalletTransaction may only be called if fBroadcastTransactions! bool RelayWalletTransaction(CConnman* connman); + /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); + std::set<uint256> GetConflicts() const; }; @@ -722,6 +721,18 @@ private: std::unique_ptr<CWalletDBWrapper> dbw; + /** + * The following is used to keep track of how far behind the wallet is + * from the chain sync, and to allow clients to block on us being caught up. + * + * Note that this is *not* how far we've processed, we may need some rescan + * to have seen all transactions in the chain, but is only used to track + * live BlockConnected callbacks. + * + * Protected by cs_main (see BlockUntilSyncedToCurrentChain) + */ + const CBlockIndex* m_last_block_processed; + public: /* * Main wallet lock. @@ -750,9 +761,11 @@ public: void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool); - // Map from Key ID (for regular keys) or Script ID (for watch-only keys) to - // key metadata. - std::map<CTxDestination, CKeyMetadata> mapKeyMetadata; + // Map from Key ID to key metadata. + std::map<CKeyID, CKeyMetadata> mapKeyMetadata; + + // Map from Script ID to key metadata (for watch-only keys). + std::map<CScriptID, CKeyMetadata> m_script_metadata; typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; @@ -817,7 +830,7 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0, const int nMinDepth = 0, const int nMaxDepth = 9999999) const; /** * Return list of available coins and locked coins grouped by non-change output address. @@ -863,7 +876,8 @@ public: //! Adds a key to the store, without saving it to disk (used by LoadWallet) bool LoadKey(const CKey& key, const CPubKey &pubkey) { return CCryptoKeyStore::AddKeyPubKey(key, pubkey); } //! Load metadata (used by LoadWallet) - bool LoadKeyMetadata(const CTxDestination& pubKey, const CKeyMetadata &metadata); + bool LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); + bool LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); bool LoadMinVersion(int nVersion) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } void UpdateTimeFirstKey(int64_t nCreateTime); @@ -919,7 +933,8 @@ public: void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); int64_t RescanFromTime(int64_t startTime, bool update); - CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); + CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate = false); + void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! @@ -1106,6 +1121,14 @@ public: caller must ensure the current wallet version is correct before calling this function). */ bool SetHDMasterKey(const CPubKey& key); + + /** + * Blocks until the wallet state is up-to-date to /at least/ the current + * chain at the time this function is entered + * Obviously holding cs_main/cs_wallet when going into this call may cause + * deadlock + */ + void BlockUntilSyncedToCurrentChain(); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index b7f873c1e4..0b0d84708d 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -3,18 +3,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/walletdb.h" - -#include "base58.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "fs.h" -#include "protocol.h" -#include "serialize.h" -#include "sync.h" -#include "util.h" -#include "utiltime.h" -#include "wallet/wallet.h" +#include <wallet/walletdb.h> + +#include <base58.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <fs.h> +#include <protocol.h> +#include <serialize.h> +#include <sync.h> +#include <util.h> +#include <utiltime.h> +#include <wallet/wallet.h> #include <atomic> @@ -268,7 +268,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CWalletTx wtx; ssValue >> wtx; CValidationState state; - if (!(CheckTransaction(wtx, state) && (wtx.GetHash() == hash) && state.IsValid())) + if (!(CheckTransaction(*wtx.tx, state) && (wtx.GetHash() == hash) && state.IsValid())) return false; // Undo serialize changes in 31600 @@ -423,27 +423,23 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } wss.fIsEncrypted = true; } - else if (strType == "keymeta" || strType == "watchmeta") + else if (strType == "keymeta") { - CTxDestination keyID; - if (strType == "keymeta") - { - CPubKey vchPubKey; - ssKey >> vchPubKey; - keyID = vchPubKey.GetID(); - } - else if (strType == "watchmeta") - { - CScript script; - ssKey >> script; - keyID = CScriptID(script); - } - + CPubKey vchPubKey; + ssKey >> vchPubKey; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - - pwallet->LoadKeyMetadata(keyID, keyMeta); + pwallet->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + } + else if (strType == "watchmeta") + { + CScript script; + ssKey >> script; + CKeyMetadata keyMeta; + ssValue >> keyMeta; + wss.nKeyMeta++; + pwallet->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == "defaultkey") { @@ -630,7 +626,6 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx) { - bool fNoncriticalErrors = false; DBErrors result = DB_LOAD_OK; try { @@ -685,9 +680,6 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal result = DB_CORRUPT; } - if (fNoncriticalErrors && result == DB_LOAD_OK) - result = DB_NONCRITICAL_ERROR; - return result; } @@ -814,14 +806,14 @@ bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDa return true; } -bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { - return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); + return CDB::VerifyEnvironment(walletFile, walletDir, errorStr); } -bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr) { - return CDB::VerifyDatabaseFile(walletFile, dataDir, warningStr, errorStr, CWalletDB::Recover); + return CDB::VerifyDatabaseFile(walletFile, walletDir, warningStr, errorStr, CWalletDB::Recover); } bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 3a146179af..e815bcfeda 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H -#include "amount.h" -#include "primitives/transaction.h" -#include "wallet/db.h" -#include "key.h" +#include <amount.h> +#include <primitives/transaction.h> +#include <wallet/db.h> +#include <key.h> #include <list> #include <stdint.h> @@ -226,9 +226,9 @@ public: /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp new file mode 100644 index 0000000000..f15e5de1e2 --- /dev/null +++ b/src/wallet/walletutil.cpp @@ -0,0 +1,27 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/walletutil.h> + +fs::path GetWalletDir() +{ + fs::path path; + + if (gArgs.IsArgSet("-walletdir")) { + path = fs::system_complete(gArgs.GetArg("-walletdir", "")); + if (!fs::is_directory(path)) { + // If the path specified doesn't exist, we return the deliberately + // invalid empty string. + path = ""; + } + } else { + path = GetDataDir(); + // If a wallets directory exists, use that, otherwise default to GetDataDir + if (fs::is_directory(path / "wallets")) { + path /= "wallets"; + } + } + + return path; +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h new file mode 100644 index 0000000000..50ff736402 --- /dev/null +++ b/src/wallet/walletutil.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_UTIL_H +#define BITCOIN_WALLET_UTIL_H + +#include <chainparamsbase.h> +#include <util.h> + +//! Get the path of the wallet directory. +fs::path GetWalletDir(); + +#endif // BITCOIN_WALLET_UTIL_H diff --git a/src/warnings.cpp b/src/warnings.cpp index d4e33b701a..c52a1fdb47 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -3,10 +3,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sync.h" -#include "clientversion.h" -#include "util.h" -#include "warnings.h" +#include <sync.h> +#include <clientversion.h> +#include <util.h> +#include <warnings.h> CCriticalSection cs_warnings; std::string strMiscWarning; diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index 9f5cb3ba67..ce5c753eb4 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "zmqabstractnotifier.h" -#include "util.h" +#include <zmq/zmqabstractnotifier.h> +#include <util.h> CZMQAbstractNotifier::~CZMQAbstractNotifier() diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index 7828822149..6fbaba2036 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H -#include "zmqconfig.h" +#include <zmq/zmqconfig.h> class CBlockIndex; class CZMQAbstractNotifier; diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h index 610d7fbda4..ce8f3e13db 100644 --- a/src/zmq/zmqconfig.h +++ b/src/zmq/zmqconfig.h @@ -6,7 +6,7 @@ #define BITCOIN_ZMQ_ZMQCONFIG_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdarg.h> @@ -16,8 +16,8 @@ #include <zmq.h> #endif -#include "primitives/block.h" -#include "primitives/transaction.h" +#include <primitives/block.h> +#include <primitives/transaction.h> void zmqError(const char *str); diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 9909395d84..b663615752 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "zmqnotificationinterface.h" -#include "zmqpublishnotifier.h" +#include <zmq/zmqnotificationinterface.h> +#include <zmq/zmqpublishnotifier.h> -#include "version.h" -#include "validation.h" -#include "streams.h" -#include "util.h" +#include <version.h> +#include <validation.h> +#include <streams.h> +#include <util.h> void zmqError(const char *str) { @@ -40,15 +40,15 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; - for (std::map<std::string, CZMQNotifierFactory>::const_iterator i=factories.begin(); i!=factories.end(); ++i) + for (const auto& entry : factories) { - std::string arg("-zmq" + i->first); + std::string arg("-zmq" + entry.first); if (gArgs.IsArgSet(arg)) { - CZMQNotifierFactory factory = i->second; + CZMQNotifierFactory factory = entry.second; std::string address = gArgs.GetArg(arg, ""); CZMQAbstractNotifier *notifier = factory(); - notifier->SetType(i->first); + notifier->SetType(entry.first); notifier->SetAddress(address); notifiers.push_back(notifier); } diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index cb92216fa4..222d31e2ad 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H #define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H -#include "validationinterface.h" +#include <validationinterface.h> #include <string> #include <map> #include <list> diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index ab54e2bb8b..acccb896c0 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "streams.h" -#include "zmqpublishnotifier.h" -#include "validation.h" -#include "util.h" -#include "rpc/server.h" +#include <chain.h> +#include <chainparams.h> +#include <streams.h> +#include <zmq/zmqpublishnotifier.h> +#include <validation.h> +#include <util.h> +#include <rpc/server.h> static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 1790fe5698..0e9bb23e5b 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H #define BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H -#include "zmqabstractnotifier.h" +#include <zmq/zmqabstractnotifier.h> class CBlockIndex; diff --git a/test/functional/README.md b/test/functional/README.md index 2558bd017d..6be4d9cfab 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -63,12 +63,12 @@ wrappers for them, `msg_block`, `msg_tx`, etc). with the bitcoind(s) being tested (using python's asyncore package); the other implements the test logic. -- `NodeConn` is the class used to connect to a bitcoind. If you implement -a callback class that derives from `NodeConnCB` and pass that to the -`NodeConn` object, your code will receive the appropriate callbacks when -events of interest arrive. +- `P2PConnection` is the class used to connect to a bitcoind. `P2PInterface` +contains the higher level logic for processing P2P payloads and connecting to +the Bitcoin Core node application logic. For custom behaviour, subclass the +P2PInterface object and override the callback methods. -- Call `NetworkThread.start()` after all `NodeConn` objects are created to +- Call `network_thread_start()` after all `P2PInterface` objects are created to start the networking thread. (Continue with the test logic in your existing thread.) diff --git a/test/functional/assumevalid.py b/test/functional/assumevalid.py index 65685c48b7..362b94e0d3 100755 --- a/test/functional/assumevalid.py +++ b/test/functional/assumevalid.py @@ -38,16 +38,16 @@ from test_framework.mininode import (CBlockHeader, CTransaction, CTxIn, CTxOut, - NetworkThread, - NodeConn, - NodeConnCB, + network_thread_join, + network_thread_start, + P2PInterface, msg_block, msg_headers) from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (p2p_port, assert_equal) +from test_framework.util import assert_equal -class BaseNode(NodeConnCB): +class BaseNode(P2PInterface): def send_header_for_blocks(self, new_blocks): headers_message = msg_headers() headers_message.headers = [CBlockHeader(b) for b in new_blocks] @@ -65,13 +65,13 @@ class AssumeValidTest(BitcoinTestFramework): # signature so we can pass in the block hash as assumevalid. self.start_node(0) - def send_blocks_until_disconnected(self, node): + def send_blocks_until_disconnected(self, p2p_conn): """Keep sending blocks to the node until we're disconnected.""" for i in range(len(self.blocks)): - if not node.connection: + if p2p_conn.state != "connected": break try: - node.send_message(msg_block(self.blocks[i])) + p2p_conn.send_message(msg_block(self.blocks[i])) except IOError as e: assert str(e) == 'Not connected, no pushbuf' break @@ -97,13 +97,10 @@ class AssumeValidTest(BitcoinTestFramework): def run_test(self): # Connect to node0 - node0 = BaseNode() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) - NetworkThread().start() # Start up network handling in another thread - node0.wait_for_verack() + network_thread_start() + self.nodes[0].p2p.wait_for_verack() # Build the blockchain self.tip = int(self.nodes[0].getbestblockhash(), 16) @@ -163,39 +160,44 @@ class AssumeValidTest(BitcoinTestFramework): self.block_time += 1 height += 1 + # We're adding new connections so terminate the network thread + self.nodes[0].disconnect_p2ps() + network_thread_join() + # Start node1 and node2 with assumevalid so they accept a block with a bad signature. self.start_node(1, extra_args=["-assumevalid=" + hex(block102.sha256)]) - node1 = BaseNode() # connects to node1 - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1)) - node1.add_connection(connections[1]) - node1.wait_for_verack() - self.start_node(2, extra_args=["-assumevalid=" + hex(block102.sha256)]) - node2 = BaseNode() # connects to node2 - connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) - node2.add_connection(connections[2]) - node2.wait_for_verack() + + p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) + p2p1 = self.nodes[1].add_p2p_connection(BaseNode()) + p2p2 = self.nodes[2].add_p2p_connection(BaseNode()) + + network_thread_start() + + p2p0.wait_for_verack() + p2p1.wait_for_verack() + p2p2.wait_for_verack() # send header lists to all three nodes - node0.send_header_for_blocks(self.blocks[0:2000]) - node0.send_header_for_blocks(self.blocks[2000:]) - node1.send_header_for_blocks(self.blocks[0:2000]) - node1.send_header_for_blocks(self.blocks[2000:]) - node2.send_header_for_blocks(self.blocks[0:200]) + p2p0.send_header_for_blocks(self.blocks[0:2000]) + p2p0.send_header_for_blocks(self.blocks[2000:]) + p2p1.send_header_for_blocks(self.blocks[0:2000]) + p2p1.send_header_for_blocks(self.blocks[2000:]) + p2p2.send_header_for_blocks(self.blocks[0:200]) # Send blocks to node0. Block 102 will be rejected. - self.send_blocks_until_disconnected(node0) + self.send_blocks_until_disconnected(p2p0) self.assert_blockchain_height(self.nodes[0], 101) # Send all blocks to node1. All blocks will be accepted. for i in range(2202): - node1.send_message(msg_block(self.blocks[i])) + p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - node1.sync_with_ping(120) + p2p1.sync_with_ping(120) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. - self.send_blocks_until_disconnected(node2) + self.send_blocks_until_disconnected(p2p2) self.assert_blockchain_height(self.nodes[2], 101) if __name__ == '__main__': diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index 2cd6df6e37..f4df879723 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -66,15 +66,12 @@ class BIP65Test(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - node0 = NodeConnCB() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + self.nodes[0].add_p2p_connection(P2PInterface()) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() self.log.info("Mining %d blocks", CLTV_HEIGHT - 2) self.coinbase_blocks = self.nodes[0].generate(CLTV_HEIGHT - 2) @@ -95,7 +92,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(self.nodes[0].getbestblockhash(), block.hash) self.log.info("Test that blocks must now be at least version 4") @@ -104,15 +101,15 @@ class BIP65Test(BitcoinTestFramework): block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time) block.nVersion = 3 block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) - assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000003)') - assert_equal(node0.last_message["reject"].data, block.sha256) - del node0.last_message["reject"] + assert_equal(self.nodes[0].p2p.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'bad-version(0x00000003)') + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + del self.nodes[0].p2p.last_message["reject"] self.log.info("Test that invalid-according-to-cltv transactions cannot appear in a block") block.nVersion = 4 @@ -125,7 +122,7 @@ class BIP65Test(BitcoinTestFramework): # First we show that this tx is valid except for CLTV by getting it # accepted to the mempool (which we can achieve with # -promiscuousmempoolflags). - node0.send_and_ping(msg_tx(spendtx)) + self.nodes[0].p2p.send_and_ping(msg_tx(spendtx)) assert spendtx.hash in self.nodes[0].getrawmempool() # Now we verify that a block with this transaction is invalid. @@ -133,18 +130,18 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] - assert_equal(node0.last_message["reject"].data, block.sha256) - if node0.last_message["reject"].code == REJECT_INVALID: + assert self.nodes[0].p2p.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + if self.nodes[0].p2p.last_message["reject"].code == REJECT_INVALID: # Generic rejection when a block is invalid - assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'block-validation-failed') else: - assert b'Negative locktime' in node0.last_message["reject"].reason + assert b'Negative locktime' in self.nodes[0].p2p.last_message["reject"].reason self.log.info("Test that a version 4 block with a valid-according-to-CLTV transaction is accepted") spendtx = cltv_validate(self.nodes[0], spendtx, CLTV_HEIGHT - 1) @@ -155,7 +152,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) diff --git a/test/functional/bip68-112-113-p2p.py b/test/functional/bip68-112-113-p2p.py index 7e6a4f4408..d3c7d8fc11 100755 --- a/test/functional/bip68-112-113-p2p.py +++ b/test/functional/bip68-112-113-p2p.py @@ -45,7 +45,7 @@ bip112tx_special - test negative argument to OP_CSV from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * -from test_framework.mininode import ToHex, CTransaction, NetworkThread +from test_framework.mininode import ToHex, CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import * @@ -100,7 +100,7 @@ class BIP68_112_113Test(ComparisonTestFramework): def run_test(self): test = TestManager(self, self.options.tmpdir) test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def send_generic_input_tx(self, node, coinbases): diff --git a/test/functional/bip9-softforks.py b/test/functional/bip9-softforks.py index 904789301a..4cd6a177aa 100755 --- a/test/functional/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -22,7 +22,7 @@ import itertools from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript, OP_1NEGATE, OP_CHECKSEQUENCEVERIFY, OP_DROP @@ -36,7 +36,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): def run_test(self): self.test = TestManager(self, self.options.tmpdir) self.test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.test.run() def create_transaction(self, node, coinbase, to_address, amount): @@ -245,8 +245,8 @@ class BIP9SoftForksTest(ComparisonTestFramework): self.setup_chain() self.setup_network() self.test.add_all_connections(self.nodes) - NetworkThread().start() - self.test.test_nodes[0].wait_for_verack() + network_thread_start() + self.test.p2p_connections[0].wait_for_verack() def get_tests(self): for test in itertools.chain( diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index c620d3e155..5d7b889e83 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -54,14 +54,12 @@ class BIP66Test(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - node0 = NodeConnCB() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) - NetworkThread().start() # Start up network handling in another thread + self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2) self.coinbase_blocks = self.nodes[0].generate(DERSIG_HEIGHT - 2) @@ -83,7 +81,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(self.nodes[0].getbestblockhash(), block.hash) self.log.info("Test that blocks must now be at least version 3") @@ -93,15 +91,15 @@ class BIP66Test(BitcoinTestFramework): block.nVersion = 2 block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) - assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000002)') - assert_equal(node0.last_message["reject"].data, block.sha256) - del node0.last_message["reject"] + assert_equal(self.nodes[0].p2p.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'bad-version(0x00000002)') + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + del self.nodes[0].p2p.last_message["reject"] self.log.info("Test that transactions with non-DER signatures cannot appear in a block") block.nVersion = 3 @@ -114,7 +112,7 @@ class BIP66Test(BitcoinTestFramework): # First we show that this tx is valid except for DERSIG by getting it # accepted to the mempool (which we can achieve with # -promiscuousmempoolflags). - node0.send_and_ping(msg_tx(spendtx)) + self.nodes[0].p2p.send_and_ping(msg_tx(spendtx)) assert spendtx.hash in self.nodes[0].getrawmempool() # Now we verify that a block with this transaction is invalid. @@ -123,23 +121,23 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: # We can receive different reject messages depending on whether # bitcoind is running with multiple script check threads. If script # check threads are not in use, then transaction script validation # happens sequentially, and bitcoind produces more specific reject # reasons. - assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] - assert_equal(node0.last_message["reject"].data, block.sha256) - if node0.last_message["reject"].code == REJECT_INVALID: + assert self.nodes[0].p2p.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + if self.nodes[0].p2p.last_message["reject"].code == REJECT_INVALID: # Generic rejection when a block is invalid - assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'block-validation-failed') else: - assert b'Non-canonical DER signature' in node0.last_message["reject"].reason + assert b'Non-canonical DER signature' in self.nodes[0].p2p.last_message["reject"].reason self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted") block.vtx[1] = create_transaction(self.nodes[0], @@ -148,7 +146,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) if __name__ == '__main__': diff --git a/test/functional/bitcoin_cli.py b/test/functional/bitcoin_cli.py index 996cbb8a12..d1cd3b3620 100755 --- a/test/functional/bitcoin_cli.py +++ b/test/functional/bitcoin_cli.py @@ -35,8 +35,11 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) assert_raises_process_error(1, "incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + self.log.info("Make sure that -getinfo with arguments fails") + assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) + self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") - cli_get_info = self.nodes[0].cli('-getinfo').help() + cli_get_info = self.nodes[0].cli().send_cli('-getinfo') wallet_info = self.nodes[0].getwalletinfo() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 4576cb036a..49fafbc9aa 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -5,6 +5,7 @@ """Test RPCs related to blockchainstate. Test the following RPCs: + - getblockchaininfo - gettxoutsetinfo - getdifficulty - getbestblockhash @@ -58,6 +59,7 @@ class BlockchainTest(BitcoinTestFramework): 'chainwork', 'difficulty', 'headers', + 'initialblockdownload', 'mediantime', 'pruned', 'size_on_disk', diff --git a/test/functional/conf_args.py b/test/functional/conf_args.py new file mode 100755 index 0000000000..61abba8082 --- /dev/null +++ b/test/functional/conf_args.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various command line arguments and configuration file parameters.""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import get_datadir_path + +class ConfArgsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + self.stop_node(0) + # Remove the -datadir argument so it doesn't override the config file + self.nodes[0].args = [arg for arg in self.nodes[0].args if not arg.startswith("-datadir")] + + default_data_dir = get_datadir_path(self.options.tmpdir, 0) + new_data_dir = os.path.join(default_data_dir, 'newdatadir') + new_data_dir_2 = os.path.join(default_data_dir, 'newdatadir2') + + # Check that using -datadir argument on non-existent directory fails + self.nodes[0].datadir = new_data_dir + self.assert_start_raises_init_error(0, ['-datadir='+new_data_dir], 'Error: Specified data directory "' + new_data_dir + '" does not exist.') + + # Check that using non-existent datadir in conf file fails + conf_file = os.path.join(default_data_dir, "bitcoin.conf") + with open(conf_file, 'a', encoding='utf8') as f: + f.write("datadir=" + new_data_dir + "\n") + self.assert_start_raises_init_error(0, ['-conf='+conf_file], 'Error reading configuration file: specified data directory "' + new_data_dir + '" does not exist.') + + # Create the directory and ensure the config file now works + os.mkdir(new_data_dir) + self.start_node(0, ['-conf='+conf_file, '-wallet=w1']) + self.stop_node(0) + assert os.path.isfile(os.path.join(new_data_dir, 'regtest', 'wallets', 'w1')) + + # Ensure command line argument overrides datadir in conf + os.mkdir(new_data_dir_2) + self.nodes[0].datadir = new_data_dir_2 + self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file, '-wallet=w2']) + assert os.path.isfile(os.path.join(new_data_dir_2, 'regtest', 'wallets', 'w2')) + +if __name__ == '__main__': + ConfArgsTest().main() diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 87d73ad14a..12be685ecf 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -17,30 +17,29 @@ from collections import defaultdict from test_framework.blocktools import (create_block, create_coinbase) from test_framework.mininode import ( CInv, - NetworkThread, - NodeConn, - NodeConnCB, + P2PInterface, mininode_lock, msg_block, msg_getdata, + network_thread_join, + network_thread_start, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - p2p_port, wait_until, ) -# NodeConnCB is a class containing callbacks to be executed when a P2P -# message is received from the node-under-test. Subclass NodeConnCB and +# P2PInterface is a class containing callbacks to be executed when a P2P +# message is received from the node-under-test. Subclass P2PInterface and # override the on_*() methods if you need custom behaviour. -class BaseNode(NodeConnCB): +class BaseNode(P2PInterface): def __init__(self): - """Initialize the NodeConnCB + """Initialize the P2PInterface Used to inialize custom properties for the Node that aren't - included by default in the base class. Be aware that the NodeConnCB + included by default in the base class. Be aware that the P2PInterface base class already stores a counter for each P2P message type and the last received message of each type, which should be sufficient for the needs of most tests. @@ -51,14 +50,14 @@ class BaseNode(NodeConnCB): # Stores a dictionary of all blocks received self.block_receive_map = defaultdict(int) - def on_block(self, conn, message): + def on_block(self, message): """Override the standard on_block callback Store the hash of a received block in the dictionary.""" message.block.calc_sha256() self.block_receive_map[message.block.sha256] += 1 - def on_inv(self, conn, message): + def on_inv(self, message): """Override the standard on_inv callback""" pass @@ -133,17 +132,14 @@ class ExampleTest(BitcoinTestFramework): def run_test(self): """Main test logic""" - # Create a P2P connection to one of the nodes - node0 = BaseNode() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + # Create P2P connections to two of the nodes + self.nodes[0].add_p2p_connection(BaseNode()) # Start up network handling in another thread. This needs to be called # after the P2P connections have been created. - NetworkThread().start() + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() # Generating a block on one of the nodes will get us out of IBD blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] @@ -179,8 +175,8 @@ class ExampleTest(BitcoinTestFramework): block = create_block(self.tip, create_coinbase(height), self.block_time) block.solve() block_message = msg_block(block) - # Send message is used to send a P2P message to the node over our NodeConn connection - node0.send_message(block_message) + # Send message is used to send a P2P message to the node over our P2PInterface + self.nodes[0].p2p.send_message(block_message) self.tip = block.sha256 blocks.append(self.tip) self.block_time += 1 @@ -193,28 +189,33 @@ class ExampleTest(BitcoinTestFramework): connect_nodes(self.nodes[1], 2) self.log.info("Add P2P connection to node2") - node2 = BaseNode() - connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) - node2.add_connection(connections[1]) - node2.wait_for_verack() + # We can't add additional P2P connections once the network thread has started. Disconnect the connection + # to node0, wait for the network thread to terminate, then connect to node2. This is specific to + # the current implementation of the network thread and may be improved in future. + self.nodes[0].disconnect_p2ps() + network_thread_join() + + self.nodes[2].add_p2p_connection(BaseNode()) + network_thread_start() + self.nodes[2].p2p.wait_for_verack() self.log.info("Wait for node2 reach current tip. Test that it has propagated all the blocks to us") getdata_request = msg_getdata() for block in blocks: getdata_request.inv.append(CInv(2, block)) - node2.send_message(getdata_request) + self.nodes[2].p2p.send_message(getdata_request) # wait_until() will loop until a predicate condition is met. Use it to test properties of the - # NodeConnCB objects. - wait_until(lambda: sorted(blocks) == sorted(list(node2.block_receive_map.keys())), timeout=5, lock=mininode_lock) + # P2PInterface objects. + wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock) self.log.info("Check that each block was received only once") - # The network thread uses a global lock on data access to the NodeConn objects when sending and receiving - # messages. The test thread should acquire the global lock before accessing any NodeConn data to avoid locking + # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving + # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. with mininode_lock: - for block in node2.block_receive_map.values(): + for block in self.nodes[2].p2p.block_receive_map.values(): assert_equal(block, 1) if __name__ == '__main__': diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py new file mode 100755 index 0000000000..da4e7b0398 --- /dev/null +++ b/test/functional/feature_logging.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test debug logging.""" + +import os + +from test_framework.test_framework import BitcoinTestFramework + +class LoggingTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + # test default log file name + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "debug.log")) + + # test alternative log file name in datadir + self.restart_node(0, ["-debuglogfile=foo.log"]) + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "foo.log")) + + # test alternative log file name outside datadir + tempname = os.path.join(self.options.tmpdir, "foo.log") + self.restart_node(0, ["-debuglogfile=%s" % tempname]) + assert os.path.isfile(tempname) + + # check that invalid log (relative) will cause error + invdir = os.path.join(self.nodes[0].datadir, "regtest", "foo") + invalidname = os.path.join("foo", "foo.log") + self.stop_node(0) + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % (invalidname)], + "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (relative) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (absolute) will cause error + self.stop_node(0) + invdir = os.path.join(self.options.tmpdir, "foo") + invalidname = os.path.join(invdir, "foo.log") + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % invalidname], + "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (absolute) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) + + +if __name__ == '__main__': + LoggingTest().main() diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index c1a42870ec..a691595f15 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -160,6 +160,18 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['ismine'], True) assert_equal(address_assert['timestamp'], timestamp) + self.log.info("Should not import an address with private key if is already imported") + result = self.nodes[1].importmulti([{ + "scriptPubKey": { + "address": address['address'] + }, + "timestamp": "now", + "keys": [ self.nodes[0].dumpprivkey(address['address']) ] + }]) + assert_equal(result[0]['success'], False) + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'The wallet already contains the private key for this address or script') + # Address + Private key + watchonly self.log.info("Should not import an address with private key and with watchonly") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) diff --git a/test/functional/invalidblockrequest.py b/test/functional/invalidblockrequest.py index 9f44b44927..a89d1d8ef2 100755 --- a/test/functional/invalidblockrequest.py +++ b/test/functional/invalidblockrequest.py @@ -15,6 +15,7 @@ from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * from test_framework.comptool import TestManager, TestInstance, RejectResult from test_framework.blocktools import * +from test_framework.mininode import network_thread_start import copy import time @@ -32,7 +33,7 @@ class InvalidBlockRequestTest(ComparisonTestFramework): test.add_all_connections(self.nodes) self.tip = None self.block_time = None - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def get_tests(self): diff --git a/test/functional/invalidtxrequest.py b/test/functional/invalidtxrequest.py index a22bd8f8cd..c60b0fce16 100755 --- a/test/functional/invalidtxrequest.py +++ b/test/functional/invalidtxrequest.py @@ -28,7 +28,7 @@ class InvalidTxRequestTest(ComparisonTestFramework): test.add_all_connections(self.nodes) self.tip = None self.block_time = None - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def get_tests(self): diff --git a/test/functional/keypool-topup.py b/test/functional/keypool-topup.py index 160a0f7ae5..e7af3c3987 100755 --- a/test/functional/keypool-topup.py +++ b/test/functional/keypool-topup.py @@ -33,7 +33,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/node1/regtest/wallet.dat", self.tmpdir + "/wallet.bak") + shutil.copyfile(self.tmpdir + "/node1/regtest/wallets/wallet.dat", self.tmpdir + "/wallet.bak") self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) @@ -56,7 +56,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallet.dat") + shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallets/wallet.dat") self.log.info("Verify keypool is restored and balance is correct") diff --git a/test/functional/listsinceblock.py b/test/functional/listsinceblock.py index 6f428388ec..67e7744bf8 100755 --- a/test/functional/listsinceblock.py +++ b/test/functional/listsinceblock.py @@ -5,7 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import assert_equal, assert_array_result, assert_raises_rpc_error class ListSinceBlockTest (BitcoinTestFramework): def set_test_params(self): @@ -16,10 +16,43 @@ class ListSinceBlockTest (BitcoinTestFramework): self.nodes[2].generate(101) self.sync_all() + self.test_no_blockhash() + self.test_invalid_blockhash() self.test_reorg() self.test_double_spend() self.test_double_send() + def test_no_blockhash(self): + txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) + blockhash, = self.nodes[2].generate(1) + self.sync_all() + + txs = self.nodes[0].listtransactions() + assert_array_result(txs, {"txid": txid}, { + "category": "receive", + "amount": 1, + "blockhash": blockhash, + "confirmations": 1, + }) + assert_equal( + self.nodes[0].listsinceblock(), + {"lastblock": blockhash, + "removed": [], + "transactions": txs}) + assert_equal( + self.nodes[0].listsinceblock(""), + {"lastblock": blockhash, + "removed": [], + "transactions": txs}) + + def test_invalid_blockhash(self): + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4") + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "0000000000000000000000000000000000000000000000000000000000000000") + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "invalid-hex") + def test_reorg(self): ''' `listsinceblock` did not behave correctly when handed a block that was diff --git a/test/functional/maxuploadtarget.py b/test/functional/maxuploadtarget.py index 1f402798e7..cf2e484d9f 100755 --- a/test/functional/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -17,15 +17,15 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.block_receive_map = defaultdict(int) - def on_inv(self, conn, message): + def on_inv(self, message): pass - def on_block(self, conn, message): + def on_block(self, message): message.block.calc_sha256() self.block_receive_map[message.block.sha256] += 1 @@ -49,19 +49,17 @@ class MaxUploadTest(BitcoinTestFramework): # Generate some old blocks self.nodes[0].generate(130) - # test_nodes[0] will only request old blocks - # test_nodes[1] will only request new blocks - # test_nodes[2] will test resetting the counters - test_nodes = [] - connections = [] + # p2p_conns[0] will only request old blocks + # p2p_conns[1] will only request new blocks + # p2p_conns[2] will test resetting the counters + p2p_conns = [] - for i in range(3): - test_nodes.append(TestNode()) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) - test_nodes[i].add_connection(connections[i]) + for _ in range(3): + p2p_conns.append(self.nodes[0].add_p2p_connection(TestNode())) - NetworkThread().start() # Start up network handling in another thread - [x.wait_for_verack() for x in test_nodes] + network_thread_start() + for p2pc in p2p_conns: + p2pc.wait_for_verack() # Test logic begins here @@ -83,7 +81,7 @@ class MaxUploadTest(BitcoinTestFramework): big_new_block = self.nodes[0].getbestblockhash() big_new_block = int(big_new_block, 16) - # test_nodes[0] will test what happens if we just keep requesting the + # p2p_conns[0] will test what happens if we just keep requesting the # the same big old block too many times (expect: disconnect) getdata_request = msg_getdata() @@ -97,34 +95,34 @@ class MaxUploadTest(BitcoinTestFramework): # 576MB will be reserved for relaying new blocks, so expect this to # succeed for ~235 tries. for i in range(success_count): - test_nodes[0].send_message(getdata_request) - test_nodes[0].sync_with_ping() - assert_equal(test_nodes[0].block_receive_map[big_old_block], i+1) + p2p_conns[0].send_message(getdata_request) + p2p_conns[0].sync_with_ping() + assert_equal(p2p_conns[0].block_receive_map[big_old_block], i+1) assert_equal(len(self.nodes[0].getpeerinfo()), 3) # At most a couple more tries should succeed (depending on how long # the test has been running so far). for i in range(3): - test_nodes[0].send_message(getdata_request) - test_nodes[0].wait_for_disconnect() + p2p_conns[0].send_message(getdata_request) + p2p_conns[0].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 2) self.log.info("Peer 0 disconnected after downloading old block too many times") - # Requesting the current block on test_nodes[1] should succeed indefinitely, + # Requesting the current block on p2p_conns[1] should succeed indefinitely, # even when over the max upload target. # We'll try 800 times getdata_request.inv = [CInv(2, big_new_block)] for i in range(800): - test_nodes[1].send_message(getdata_request) - test_nodes[1].sync_with_ping() - assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + p2p_conns[1].send_message(getdata_request) + p2p_conns[1].sync_with_ping() + assert_equal(p2p_conns[1].block_receive_map[big_new_block], i+1) self.log.info("Peer 1 able to repeatedly download new block") - # But if test_nodes[1] tries for an old block, it gets disconnected too. + # But if p2p_conns[1] tries for an old block, it gets disconnected too. getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[1].send_message(getdata_request) - test_nodes[1].wait_for_disconnect() + p2p_conns[1].send_message(getdata_request) + p2p_conns[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) self.log.info("Peer 1 disconnected after trying to download old block") @@ -132,39 +130,37 @@ class MaxUploadTest(BitcoinTestFramework): self.log.info("Advancing system time on node to clear counters...") # If we advance the time by 24 hours, then the counters should reset, - # and test_nodes[2] should be able to retrieve the old block. + # and p2p_conns[2] should be able to retrieve the old block. self.nodes[0].setmocktime(int(time.time())) - test_nodes[2].sync_with_ping() - test_nodes[2].send_message(getdata_request) - test_nodes[2].sync_with_ping() - assert_equal(test_nodes[2].block_receive_map[big_old_block], 1) + p2p_conns[2].sync_with_ping() + p2p_conns[2].send_message(getdata_request) + p2p_conns[2].sync_with_ping() + assert_equal(p2p_conns[2].block_receive_map[big_old_block], 1) self.log.info("Peer 2 able to download old block") - [c.disconnect_node() for c in connections] + self.nodes[0].disconnect_p2ps() #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 self.log.info("Restarting nodes with -whitelist=127.0.0.1") self.stop_node(0) self.start_node(0, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) - #recreate/reconnect a test node - test_nodes = [TestNode()] - connections = [NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[0])] - test_nodes[0].add_connection(connections[0]) + # Reconnect to self.nodes[0] + self.nodes[0].add_p2p_connection(TestNode()) - NetworkThread().start() # Start up network handling in another thread - test_nodes[0].wait_for_verack() + network_thread_start() + self.nodes[0].p2p.wait_for_verack() #retrieve 20 blocks which should be enough to break the 1MB limit getdata_request.inv = [CInv(2, big_new_block)] for i in range(20): - test_nodes[0].send_message(getdata_request) - test_nodes[0].sync_with_ping() - assert_equal(test_nodes[0].block_receive_map[big_new_block], i+1) + self.nodes[0].p2p.send_message(getdata_request) + self.nodes[0].p2p.sync_with_ping() + assert_equal(self.nodes[0].p2p.block_receive_map[big_new_block], i+1) getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[0].send_and_ping(getdata_request) + self.nodes[0].p2p.send_and_ping(getdata_request) assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist self.log.info("Peer still connected after trying to download old block (whitelisted)") diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 92f66be2ff..31a96ec60e 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -57,21 +57,27 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Send 5 transactions from node2 (to its own address)") for i in range(5): self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + node2_balance = self.nodes[2].getbalance() self.sync_all() self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools") assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) - self.log.debug("Stop-start node0 and node1. Verify that node0 has the transactions in its mempool and node1 does not.") + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() self.start_node(0) self.start_node(1) + self.start_node(2) # Give bitcoind a second to reload the mempool time.sleep(1) wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + wait_until(lambda: len(self.nodes[2].getrawmempool()) == 5) assert_equal(len(self.nodes[1].getrawmempool()), 0) + # Verify accounting of mempool transactions after restart is correct + assert_equal(node2_balance, self.nodes[2].getbalance()) + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py index c7579d2548..90a3de0e0d 100755 --- a/test/functional/minchainwork.py +++ b/test/functional/minchainwork.py @@ -18,7 +18,7 @@ only succeeds past a given node once its nMinimumChainWork has been exceeded. import time from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import sync_blocks, connect_nodes, assert_equal +from test_framework.util import connect_nodes, assert_equal # 2 hashes required per regtest block (with no difficulty adjustment) REGTEST_WORK_PER_BLOCK = 2 @@ -27,6 +27,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [[], ["-minimumchainwork=0x65"], ["-minimumchainwork=0x65"]] self.node_min_work = [0, 101, 101] @@ -74,6 +75,13 @@ class MinimumChainWorkTest(BitcoinTestFramework): self.nodes[0].generate(1) self.log.info("Verifying nodes are all synced") + + # Because nodes in regtest are all manual connections (eg using + # addnode), node1 should not have disconnected node0. If not for that, + # we'd expect node1 to have disconnected node0 for serving an + # insufficient work chain, in which case we'd need to reconnect them to + # continue the test. + self.sync_all() self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 6adcc1fd88..d0c40e5446 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -7,6 +7,7 @@ Verify that a bitcoind node can load multiple wallet files """ import os +import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error @@ -15,10 +16,10 @@ class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3']] + self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w']] def run_test(self): - assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3"}) + assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3", "w"}) self.stop_node(0) @@ -26,18 +27,50 @@ class MultiWalletTest(BitcoinTestFramework): self.assert_start_raises_init_error(0, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if wallet file is a directory - os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) + wallet_dir = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'wallets') + os.mkdir(os.path.join(wallet_dir, 'w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') + # should not initialize if one wallet is a copy of another + shutil.copyfile(os.path.join(wallet_dir, 'w2'), os.path.join(wallet_dir, 'w22')) + self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') + # should not initialize if wallet file is a symlink - os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) + os.symlink(os.path.join(wallet_dir, 'w1'), os.path.join(wallet_dir, 'w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') + # should not initialize if the specified walletdir does not exist + self.assert_start_raises_init_error(0, ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') + # should not initialize if the specified walletdir is not a directory + not_a_dir = os.path.join(wallet_dir, 'notadir') + open(not_a_dir, 'a').close() + self.assert_start_raises_init_error(0, ['-walletdir='+not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + + # if wallets/ doesn't exist, datadir should be the default wallet dir + wallet_dir2 = os.path.join(self.options.tmpdir, 'node0', 'regtest', 'walletdir') + os.rename(wallet_dir, wallet_dir2) + self.start_node(0, ['-wallet=w4', '-wallet=w5']) + assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) + w5 = self.nodes[0].get_wallet_rpc("w5") + w5.generate(1) + self.stop_node(0) + + # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded + os.rename(wallet_dir2, wallet_dir) + self.start_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + os.path.join(self.options.tmpdir, 'node0', 'regtest')]) + assert_equal(set(self.nodes[0].listwallets()), {"w4", "w5"}) + w5 = self.nodes[0].get_wallet_rpc("w5") + w5_info = w5.getwalletinfo() + assert_equal(w5_info['immature_balance'], 50) + + self.stop_node(0) + self.start_node(0, self.extra_args[0]) w1 = self.nodes[0].get_wallet_rpc("w1") w2 = self.nodes[0].get_wallet_rpc("w2") w3 = self.nodes[0].get_wallet_rpc("w3") + w4 = self.nodes[0].get_wallet_rpc("w") wallet_bad = self.nodes[0].get_wallet_rpc("bad") w1.generate(1) @@ -63,18 +96,26 @@ class MultiWalletTest(BitcoinTestFramework): w3_name = w3.getwalletinfo()['walletname'] assert_equal(w3_name, "w3") - assert_equal({"w1", "w2", "w3"}, {w1_name, w2_name, w3_name}) + w4_name = w4.getwalletinfo()['walletname'] + assert_equal(w4_name, "w") w1.generate(101) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) + assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) + w1.sendtoaddress(w4.getnewaddress(), 3) w1.generate(1) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) + assert_equal(w4.getbalance(), 3) + + batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) + assert_equal(batch[0]["result"]["chain"], "regtest") + assert_equal(batch[1]["result"]["walletname"], "w1") if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/node_network_limited.py b/test/functional/node_network_limited.py new file mode 100755 index 0000000000..70415e0168 --- /dev/null +++ b/test/functional/node_network_limited.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests NODE_NETWORK_LIMITED. + +Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correctly +and that it responds to getdata requests for blocks correctly: + - send a block within 288 + 2 of the tip + - disconnect peers who request blocks older than that.""" +from test_framework.messages import CInv, msg_getdata +from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, NetworkThread, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class P2PIgnoreInv(P2PInterface): + def on_inv(self, message): + # The node will send us invs for other blocks. Ignore them. + pass + + def send_getdata_for_block(self, blockhash): + getdata_request = msg_getdata() + getdata_request.inv.append(CInv(2, int(blockhash, 16))) + self.send_message(getdata_request) + +class NodeNetworkLimitedTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-prune=550']] + + def run_test(self): + node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) + NetworkThread().start() + node.wait_for_verack() + + expected_services = NODE_BLOOM | NODE_WITNESS | NODE_NETWORK_LIMITED + + self.log.info("Check that node has signalled expected services.") + assert_equal(node.nServices, expected_services) + + self.log.info("Check that the localservices is as expected.") + assert_equal(int(self.nodes[0].getnetworkinfo()['localservices'], 16), expected_services) + + self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") + blocks = self.nodes[0].generate(292) + + self.log.info("Make sure we can max retrive block at tip-288.") + node.send_getdata_for_block(blocks[1]) # last block in valid range + node.wait_for_block(int(blocks[1], 16), timeout=3) + + self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") + node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit + node.wait_for_disconnect(5) + +if __name__ == '__main__': + NodeNetworkLimitedTest().main() diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 91c4550653..9f9f2f90c0 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -15,7 +15,7 @@ Generate 427 more blocks. from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block, add_witness_commitment from test_framework.script import CScript from io import BytesIO @@ -40,7 +40,9 @@ class NULLDUMMYTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness']] + # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through + # normal segwit activation here (and don't use the default always-on behaviour). + self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness', '-vbparams=segwit:0:999999999999']] def run_test(self): self.address = self.nodes[0].getnewaddress() @@ -48,7 +50,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.wit_address = self.nodes[0].addwitnessaddress(self.address) self.wit_ms_address = self.nodes[0].addwitnessaddress(self.ms_address) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 coinbase_txid = [] for i in self.coinbase_blocks: diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 5b6429b410..bb204322ed 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -4,37 +4,32 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Since behavior differs when receiving unrequested blocks from whitelisted peers -versus non-whitelisted peers, this tests the behavior of both (effectively two -separate tests running in parallel). +Setup: two nodes, node0+node1, not connected to each other. Node1 will have +nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. -Setup: two nodes, node0 and node1, not connected to each other. Node0 does not -whitelist localhost, but node1 does. They will each be on their own chain for -this test. - -We have one NodeConn connection to each, test_node and white_node respectively. +We have one P2PInterface connection to node0 called test_node, and one to node1 +called min_work_node. The test: 1. Generate one block on each node, to leave IBD. 2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance. + The tip should advance for node0, but node1 should skip processing due to + nMinimumChainWork. + +Node1 is unused in tests 3-7: -3. Mine a block that forks the previous block, and deliver to each node from - corresponding peer. - Node0 should not process this block (just accept the header), because it is - unrequested and doesn't have more work than the tip. - Node1 should process because this is coming from a whitelisted peer. +3. Mine a block that forks from the genesis block, and deliver to test_node. + Node0 should not process this block (just accept the header), because it + is unrequested and doesn't have more or equal work to the tip. -4. Send another block that builds on the forking block. - Node0 should process this block but be stuck on the shorter chain, because - it's missing an intermediate block. - Node1 should reorg to this longer chain. +4a,b. Send another two blocks that build on the forking block. + Node0 should process the second block but be stuck on the shorter chain, + because it's missing an intermediate block. -4b.Send 288 more blocks on the longer chain. +4c.Send 288 more blocks on the longer chain (the number of blocks ahead + we currently store). Node0 should process all but the last block (too far ahead in height). - Send all headers to Node1, and then send the last block in that chain. - Node1 should accept the block because it's coming from a whitelisted peer. 5. Send a duplicate of the block in #3 to Node0. Node0 should not process the block because it is unrequested, and stay on @@ -46,13 +41,21 @@ The test: 7. Send Node0 the missing block again. Node0 should process and the tip should advance. + +8. Create a fork which is invalid at a height longer than the current chain + (ie to which the node will try to reorg) but which has headers built on top + of the invalid block. Check that we get disconnected if we send more headers + on the chain the node now knows to be invalid. + +9. Test Node1 is able to sync when connected to node0 (which should have sufficient + work on its chain). """ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import time -from test_framework.blocktools import create_block, create_coinbase +from test_framework.blocktools import create_block, create_coinbase, create_transaction class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): @@ -63,37 +66,35 @@ class AcceptBlockTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [[], ["-whitelist=127.0.0.1"]] + self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. + # Node2 will be used for non-whitelisted peers to test the interaction + # with nMinimumChainWork. self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = NodeConnCB() # connects to node0 (not whitelisted) - white_node = NodeConnCB() # connects to node1 (whitelisted) - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], white_node)) - test_node.add_connection(connections[0]) - white_node.add_connection(connections[1]) + # test_node connects to node0 (not whitelisted) + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + # min_work_node connects to node1 (whitelisted) + min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here test_node.wait_for_verack() - white_node.wait_for_verack() + min_work_node.wait_for_verack() - # 1. Have both nodes mine a block (leave IBD) + # 1. Have nodes mine a block (leave IBD) [ n.generate(1) for n in self.nodes ] tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. - # This should be accepted. + # This should be accepted by node0 blocks_h2 = [] # the height 2 blocks on each node's chain block_time = int(time.time()) + 1 for i in range(2): @@ -101,95 +102,120 @@ class AcceptBlockTest(BitcoinTestFramework): blocks_h2[i].solve() block_time += 1 test_node.send_message(msg_block(blocks_h2[0])) - white_node.send_message(msg_block(blocks_h2[1])) + min_work_node.send_message(msg_block(blocks_h2[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, min_work_node]: + x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - assert_equal(self.nodes[1].getblockcount(), 2) - self.log.info("First height 2 block accepted by both nodes") + assert_equal(self.nodes[1].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0; correctly rejected by node1") - # 3. Send another block that builds on the original tip. - blocks_h2f = [] # Blocks at height 2 that fork off the main chain - for i in range(2): - blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1)) - blocks_h2f[i].solve() - test_node.send_message(msg_block(blocks_h2f[0])) - white_node.send_message(msg_block(blocks_h2f[1])) + # 3. Send another block that builds on genesis. + block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) + block_time += 1 + block_h1f.solve() + test_node.send_message(msg_block(block_h1f)) - [ x.sync_with_ping() for x in [test_node, white_node] ] + test_node.sync_with_ping() + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h2f[0].hash: + if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) - for x in self.nodes[1].getchaintips(): - if x['hash'] == blocks_h2f[1].hash: - assert_equal(x['status'], "valid-headers") + # 4. Send another two block that build on the fork. + block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) + block_time += 1 + block_h2f.solve() + test_node.send_message(msg_block(block_h2f)) - self.log.info("Second height 2 block accepted only from whitelisted peer") + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h2f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) - # 4. Now send another block that builds on the forking chain. - blocks_h3 = [] - for i in range(2): - blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1)) - blocks_h3[i].solve() - test_node.send_message(msg_block(blocks_h3[0])) - white_node.send_message(msg_block(blocks_h3[1])) + # But this block should be accepted by node since it has equal work. + self.nodes[0].getblock(block_h2f.hash) + self.log.info("Second height 2 block accepted, but not reorg'ed to") + + # 4b. Now send another block that builds on the forking chain. + block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) + block_h3.solve() + test_node.send_message(msg_block(block_h3)) - [ x.sync_with_ping() for x in [test_node, white_node] ] - # Since the earlier block was not processed by node0, the new block + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block # can't be fully validated. + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h3[0].hash: + if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + self.nodes[0].getblock(block_h3.hash) + + # But this block should be accepted by node since it has more work. + self.nodes[0].getblock(block_h3.hash) + self.log.info("Unrequested more-work block accepted") + + # 4c. Now mine 288 more blocks and deliver; all should be processed but + # the last (height-too-high) on node (as long as its not missing any headers) + tip = block_h3 + all_blocks = [] + for i in range(288): + next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) + next_block.solve() + all_blocks.append(next_block) + tip = next_block + + # Now send the block at height 5 and check that it wasn't accepted (missing header) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) - # But this block should be accepted by node0 since it has more work. - self.nodes[0].getblock(blocks_h3[0].hash) - self.log.info("Unrequested more-work block accepted from non-whitelisted peer") + # The block at height 5 should be accepted if we provide the missing header, though + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(all_blocks[0])) + test_node.send_message(headers_message) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + self.nodes[0].getblock(all_blocks[1].hash) - # Node1 should have accepted and reorged. - assert_equal(self.nodes[1].getblockcount(), 3) - self.log.info("Successfully reorged to length 3 chain from whitelisted peer") + # Now send the blocks in all_blocks + for i in range(288): + test_node.send_message(msg_block(all_blocks[i])) + test_node.sync_with_ping() - # 4b. Now mine 288 more blocks and deliver; all should be processed but - # the last (height-too-high) on node0. Node1 should process the tip if - # we give it the headers chain leading to the tip. - tips = blocks_h3 - headers_message = msg_headers() - all_blocks = [] # node0's blocks - for j in range(2): - for i in range(288): - next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1) - next_block.solve() - if j==0: - test_node.send_message(msg_block(next_block)) - all_blocks.append(next_block) - else: - headers_message.headers.append(CBlockHeader(next_block)) - tips[j] = next_block - - time.sleep(2) # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) - headers_message.headers.pop() # Ensure the last block is unrequested - white_node.send_message(headers_message) # Send headers leading to tip - white_node.send_message(msg_block(tips[1])) # Now deliver the tip - white_node.sync_with_ping() - self.nodes[1].getblock(tips[1].hash) - self.log.info("Unrequested block far ahead of tip accepted from whitelisted peer") - # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more # work). - test_node.send_message(msg_block(blocks_h2f[0])) - # Here, if the sleep is too short, the test could falsely succeed (if the - # node hasn't processed the block by the time the sleep returns, and then - # the node processes it and incorrectly advances the tip). - # But this would be caught later on, when we verify that an inv triggers - # a getdata request for this block. + # The node should have requested the blocks at some point, so + # disconnect/reconnect first + + self.nodes[0].disconnect_p2ps() + self.nodes[1].disconnect_p2ps() + network_thread_join() + + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + test_node.wait_for_verack() + + test_node.send_message(msg_block(block_h1f)) + test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) self.log.info("Unrequested block that would complete more-work chain was ignored") @@ -200,24 +226,98 @@ class AcceptBlockTest(BitcoinTestFramework): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) + test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block - assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) + assert_equal(getdata.inv[0].hash, block_h1f.sha256) self.log.info("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) - test_node.send_message(msg_block(blocks_h2f[0])) + test_node.send_message(msg_block(block_h1f)) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) + self.nodes[0].getblock(all_blocks[286].hash) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain from non-whitelisted peer") - [ c.disconnect_node() for c in connections ] + # 8. Create a chain which is invalid at a height longer than the + # current chain, but which has more blocks on top of that + block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) + block_289f.solve() + block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) + block_290f.solve() + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) + # block_291 spends a coinbase below maturity! + block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) + block_291.hashMerkleRoot = block_291.calc_merkle_root() + block_291.solve() + block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) + block_292.solve() + + # Now send all the headers on the chain and enough blocks to trigger reorg + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_289f)) + headers_message.headers.append(CBlockHeader(block_290f)) + headers_message.headers.append(CBlockHeader(block_291)) + headers_message.headers.append(CBlockHeader(block_292)) + test_node.send_message(headers_message) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_292.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + + test_node.send_message(msg_block(block_289f)) + test_node.send_message(msg_block(block_290f)) + + test_node.sync_with_ping() + self.nodes[0].getblock(block_289f.hash) + self.nodes[0].getblock(block_290f.hash) + + test_node.send_message(msg_block(block_291)) + + # At this point we've sent an obviously-bogus block, wait for full processing + # without assuming whether we will be disconnected or not + try: + # Only wait a short while so the test doesn't take forever if we do get + # disconnected + test_node.sync_with_ping(timeout=1) + except AssertionError: + test_node.wait_for_disconnect() + + self.nodes[0].disconnect_p2ps() + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() + test_node.wait_for_verack() + + # We should have failed reorg and switched back to 290 (but have block 291) + assert_equal(self.nodes[0].getblockcount(), 290) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) + + # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected + block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) + block_293.solve() + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_293)) + test_node.send_message(headers_message) + test_node.wait_for_disconnect() + + # 9. Connect node1 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 1) + sync_blocks([self.nodes[0], self.nodes[1]]) + self.log.info("Successfully synced nodes 1 and 0") if __name__ == '__main__': AcceptBlockTest().main() diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index 94513d3f43..1e763df2a4 100755 --- a/test/functional/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -15,7 +15,7 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.script import CScript, OP_TRUE # TestNode: A peer we use to send messages to bitcoind, and store responses. -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.last_sendcmpct = [] @@ -25,21 +25,21 @@ class TestNode(NodeConnCB): # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() - def on_sendcmpct(self, conn, message): + def on_sendcmpct(self, message): self.last_sendcmpct.append(message) - def on_cmpctblock(self, conn, message): + def on_cmpctblock(self, message): self.block_announced = True self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) - def on_headers(self, conn, message): + def on_headers(self, message): self.block_announced = True for x in self.last_message["headers"].headers: x.calc_sha256() self.announced_blockhashes.add(x.sha256) - def on_inv(self, conn, message): + def on_inv(self, message): for x in self.last_message["inv"].inv: if x.type == 2: self.block_announced = True @@ -60,7 +60,7 @@ class TestNode(NodeConnCB): msg = msg_getheaders() msg.locator.vHave = locator msg.hashstop = hashstop - self.connection.send_message(msg) + self.send_message(msg) def send_header_for_blocks(self, new_blocks): headers_message = msg_headers() @@ -86,14 +86,16 @@ class TestNode(NodeConnCB): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: not self.connected, timeout=timeout, lock=mininode_lock) + wait_until(lambda: self.state != "connected", timeout=timeout, lock=mininode_lock) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True # Node0 = pre-segwit, node1 = segwit-aware self.num_nodes = 2 - self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]] + # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). + # TODO: Rewrite this test to support SegWit being always active. + self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] self.utxos = [] def build_block_on_tip(self, node, segwit=False): @@ -786,23 +788,12 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.test_node = TestNode() - self.segwit_node = TestNode() - self.old_node = TestNode() # version 1 peer <--> segwit node - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], - self.segwit_node, services=NODE_NETWORK|NODE_WITNESS)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], - self.old_node, services=NODE_NETWORK)) - self.test_node.add_connection(connections[0]) - self.segwit_node.add_connection(connections[1]) - self.old_node.add_connection(connections[2]) - - NetworkThread().start() # Start up network handling in another thread - - # Test logic begins here + self.test_node = self.nodes[0].add_p2p_connection(TestNode()) + self.segwit_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + self.old_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK) + + network_thread_start() + self.test_node.wait_for_verack() # We will need UTXOs to construct transactions in later tests. diff --git a/test/functional/p2p-feefilter.py b/test/functional/p2p-feefilter.py index 8c92365ced..ff4bed0efd 100755 --- a/test/functional/p2p-feefilter.py +++ b/test/functional/p2p-feefilter.py @@ -22,12 +22,12 @@ def allInvsMatch(invsExpected, testnode): time.sleep(1) return False -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.txinvs = [] - def on_inv(self, conn, message): + def on_inv(self, message): for i in message.inv: if (i.type == 1): self.txinvs.append(hashToHex(i.hash)) @@ -48,25 +48,23 @@ class FeeFilterTest(BitcoinTestFramework): sync_blocks(self.nodes) # Setup the p2p connections and start up the network thread. - test_node = TestNode() - connection = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) - test_node.add_connection(connection) - NetworkThread().start() - test_node.wait_for_verack() + self.nodes[0].add_p2p_connection(TestNode()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte - test_node.send_and_ping(msg_feefilter(15000)) + self.nodes[0].p2p.send_and_ping(msg_feefilter(15000)) # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) @@ -82,14 +80,14 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again - test_node.send_and_ping(msg_feefilter(0)) + self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() if __name__ == '__main__': FeeFilterTest().main() diff --git a/test/functional/p2p-fingerprint.py b/test/functional/p2p-fingerprint.py index fe60c6cd46..93ef73e25e 100755 --- a/test/functional/p2p-fingerprint.py +++ b/test/functional/p2p-fingerprint.py @@ -13,19 +13,17 @@ import time from test_framework.blocktools import (create_block, create_coinbase) from test_framework.mininode import ( CInv, - NetworkThread, - NodeConn, - NodeConnCB, + P2PInterface, msg_headers, msg_block, msg_getdata, msg_getheaders, + network_thread_start, wait_until, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - p2p_port, ) class P2PFingerprintTest(BitcoinTestFramework): @@ -77,13 +75,9 @@ class P2PFingerprintTest(BitcoinTestFramework): # This does not currently test that stale blocks timestamped within the # last month but that have over a month's worth of work are also withheld. def run_test(self): - node0 = NodeConnCB() + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) - - NetworkThread().start() + network_thread_start() node0.wait_for_verack() # Set node time to 60 days ago diff --git a/test/functional/p2p-fullblocktest.py b/test/functional/p2p-fullblocktest.py index 1d969fc7c1..010dbdccad 100755 --- a/test/functional/p2p-fullblocktest.py +++ b/test/functional/p2p-fullblocktest.py @@ -18,9 +18,10 @@ from test_framework.blocktools import * import time from test_framework.key import CECKey from test_framework.script import * +from test_framework.mininode import network_thread_start import struct -class PreviousSpendableOutput(object): +class PreviousSpendableOutput(): def __init__(self, tx = CTransaction(), n = -1): self.tx = tx self.n = n # the output we're spending @@ -68,7 +69,7 @@ class FullBlockTest(ComparisonTestFramework): def run_test(self): self.test = TestManager(self, self.options.tmpdir) self.test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.test.run() def add_transactions_to_block(self, block, tx_list): diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py index 1dc8f72cd6..ce4e6e9144 100755 --- a/test/functional/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -20,7 +20,7 @@ from test_framework.util import * banscore = 10 -class CLazyNode(NodeConnCB): +class CLazyNode(P2PInterface): def __init__(self): super().__init__() self.unexpected_msg = False @@ -30,44 +30,42 @@ class CLazyNode(NodeConnCB): self.unexpected_msg = True self.log.info("should not have received message: %s" % message.command) - def on_open(self, conn): - self.connected = True + def on_open(self): self.ever_connected = True - def on_version(self, conn, message): self.bad_message(message) - def on_verack(self, conn, message): self.bad_message(message) - def on_reject(self, conn, message): self.bad_message(message) - def on_inv(self, conn, message): self.bad_message(message) - def on_addr(self, conn, message): self.bad_message(message) - def on_alert(self, conn, message): self.bad_message(message) - def on_getdata(self, conn, message): self.bad_message(message) - def on_getblocks(self, conn, message): self.bad_message(message) - def on_tx(self, conn, message): self.bad_message(message) - def on_block(self, conn, message): self.bad_message(message) - def on_getaddr(self, conn, message): self.bad_message(message) - def on_headers(self, conn, message): self.bad_message(message) - def on_getheaders(self, conn, message): self.bad_message(message) - def on_ping(self, conn, message): self.bad_message(message) - def on_mempool(self, conn): self.bad_message(message) - def on_pong(self, conn, message): self.bad_message(message) - def on_feefilter(self, conn, message): self.bad_message(message) - def on_sendheaders(self, conn, message): self.bad_message(message) - def on_sendcmpct(self, conn, message): self.bad_message(message) - def on_cmpctblock(self, conn, message): self.bad_message(message) - def on_getblocktxn(self, conn, message): self.bad_message(message) - def on_blocktxn(self, conn, message): self.bad_message(message) + def on_version(self, message): self.bad_message(message) + def on_verack(self, message): self.bad_message(message) + def on_reject(self, message): self.bad_message(message) + def on_inv(self, message): self.bad_message(message) + def on_addr(self, message): self.bad_message(message) + def on_getdata(self, message): self.bad_message(message) + def on_getblocks(self, message): self.bad_message(message) + def on_tx(self, message): self.bad_message(message) + def on_block(self, message): self.bad_message(message) + def on_getaddr(self, message): self.bad_message(message) + def on_headers(self, message): self.bad_message(message) + def on_getheaders(self, message): self.bad_message(message) + def on_ping(self, message): self.bad_message(message) + def on_mempool(self, message): self.bad_message(message) + def on_pong(self, message): self.bad_message(message) + def on_feefilter(self, message): self.bad_message(message) + def on_sendheaders(self, message): self.bad_message(message) + def on_sendcmpct(self, message): self.bad_message(message) + def on_cmpctblock(self, message): self.bad_message(message) + def on_getblocktxn(self, message): self.bad_message(message) + def on_blocktxn(self, message): self.bad_message(message) # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. class CNodeNoVersionBan(CLazyNode): # send a bunch of veracks without sending a message. This should get us disconnected. # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes - def on_open(self, conn): - super().on_open(conn) + def on_open(self): + super().on_open() for i in range(banscore): self.send_message(msg_verack()) - def on_reject(self, conn, message): pass + def on_reject(self, message): pass # Node that never sends a version. This one just sits idle and hopes to receive # any message (it shouldn't!) @@ -81,15 +79,15 @@ class CNodeNoVerackIdle(CLazyNode): self.version_received = False super().__init__() - def on_reject(self, conn, message): pass - def on_verack(self, conn, message): pass + def on_reject(self, message): pass + def on_verack(self, message): pass # When version is received, don't reply with a verack. Instead, see if the # node will give us a message that it shouldn't. This is not an exhaustive # list! - def on_version(self, conn, message): + def on_version(self, message): self.version_received = True - conn.send_message(msg_ping()) - conn.send_message(msg_getaddr()) + self.send_message(msg_ping()) + self.send_message(msg_getaddr()) class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): @@ -97,26 +95,15 @@ class P2PLeakTest(BitcoinTestFramework): self.extra_args = [['-banscore='+str(banscore)]] def run_test(self): - no_version_bannode = CNodeNoVersionBan() - no_version_idlenode = CNodeNoVersionIdle() - no_verack_idlenode = CNodeNoVerackIdle() - unsupported_service_bit5_node = CLazyNode() - unsupported_service_bit7_node = CLazyNode() - self.nodes[0].setmocktime(1501545600) # August 1st 2017 - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_bannode, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_idlenode, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_verack_idlenode)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) - no_version_bannode.add_connection(connections[0]) - no_version_idlenode.add_connection(connections[1]) - no_verack_idlenode.add_connection(connections[2]) - unsupported_service_bit5_node.add_connection(connections[3]) - unsupported_service_bit7_node.add_connection(connections[4]) - - NetworkThread().start() # Start up network handling in another thread + + no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False) + no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False) + no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle()) + unsupported_service_bit5_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + unsupported_service_bit7_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) + + network_thread_start() wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) @@ -131,16 +118,17 @@ class P2PLeakTest(BitcoinTestFramework): time.sleep(5) #This node should have been banned - assert not no_version_bannode.connected + assert no_version_bannode.state != "connected" # These nodes should have been disconnected - assert not unsupported_service_bit5_node.connected - assert not unsupported_service_bit7_node.connected + assert unsupported_service_bit5_node.state != "connected" + assert unsupported_service_bit7_node.state != "connected" - [conn.disconnect_node() for conn in connections] + self.nodes[0].disconnect_p2ps() - # Wait until all connections are closed + # Wait until all connections are closed and the network thread has terminated wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) + network_thread_join() # Make sure no unexpected messages came in assert(no_version_bannode.unexpected_msg == False) @@ -152,15 +140,11 @@ class P2PLeakTest(BitcoinTestFramework): self.log.info("Service bits 5 and 7 are allowed after August 1st 2018") self.nodes[0].setmocktime(1533168000) # August 2nd 2018 - allowed_service_bit5_node = NodeConnCB() - allowed_service_bit7_node = NodeConnCB() - - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) - allowed_service_bit5_node.add_connection(connections[5]) - allowed_service_bit7_node.add_connection(connections[6]) + allowed_service_bit5_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + allowed_service_bit7_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) - NetworkThread().start() # Network thread stopped when all previous NodeConnCBs disconnected. Restart it + # Network thread stopped when all previous P2PInterfaces disconnected. Restart it + network_thread_start() wait_until(lambda: allowed_service_bit5_node.message_count["verack"], lock=mininode_lock) wait_until(lambda: allowed_service_bit7_node.message_count["verack"], lock=mininode_lock) diff --git a/test/functional/p2p-mempool.py b/test/functional/p2p-mempool.py index 40fcde2605..168f9f685a 100755 --- a/test/functional/p2p-mempool.py +++ b/test/functional/p2p-mempool.py @@ -19,16 +19,14 @@ class P2PMempoolTests(BitcoinTestFramework): self.extra_args = [["-peerbloomfilters=0"]] def run_test(self): - #connect a mininode - aTestNode = NodeConnCB() - node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) - aTestNode.add_connection(node) - NetworkThread().start() - aTestNode.wait_for_verack() + # Add a p2p connection + self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() #request mempool - aTestNode.send_message(msg_mempool()) - aTestNode.wait_for_disconnect() + self.nodes[0].p2p.send_message(msg_mempool()) + self.nodes[0].p2p.wait_for_disconnect() #mininode must be disconnected at this point assert_equal(len(self.nodes[0].getpeerinfo()), 0) diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index a9ef47559b..a06601c38e 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -31,12 +31,40 @@ def get_virtual_size(witness_block): vsize = int((3*base_size + total_size + 3)/4) return vsize -class TestNode(NodeConnCB): +def test_transaction_acceptance(rpc, p2p, tx, with_witness, accepted, reason=None): + """Send a transaction to the node and check that it's accepted to the mempool + + - Submit the transaction over the p2p interface + - use the getrawmempool rpc to check for acceptance.""" + tx_message = msg_tx(tx) + if with_witness: + tx_message = msg_witness_tx(tx) + p2p.send_message(tx_message) + p2p.sync_with_ping() + assert_equal(tx.hash in rpc.getrawmempool(), accepted) + if (reason != None and not accepted): + # Check the rejection reason as well. + with mininode_lock: + assert_equal(p2p.last_message["reject"].reason, reason) + +def test_witness_block(rpc, p2p, block, accepted, with_witness=True): + """Send a block to the node and check that it's accepted + + - Submit the block over the p2p interface + - use the getbestblockhash rpc to check for acceptance.""" + if with_witness: + p2p.send_message(msg_witness_block(block)) + else: + p2p.send_message(msg_block(block)) + p2p.sync_with_ping() + assert_equal(rpc.getbestblockhash() == block.hash, accepted) + +class TestNode(P2PInterface): def __init__(self): super().__init__() self.getdataset = set() - def on_getdata(self, conn, message): + def on_getdata(self, message): for inv in message.inv: self.getdataset.add(inv.hash) @@ -67,29 +95,8 @@ class TestNode(NodeConnCB): self.wait_for_block(blockhash, timeout) return self.last_message["block"].block - def test_transaction_acceptance(self, tx, with_witness, accepted, reason=None): - tx_message = msg_tx(tx) - if with_witness: - tx_message = msg_witness_tx(tx) - self.send_message(tx_message) - self.sync_with_ping() - assert_equal(tx.hash in self.connection.rpc.getrawmempool(), accepted) - if (reason != None and not accepted): - # Check the rejection reason as well. - with mininode_lock: - assert_equal(self.last_message["reject"].reason, reason) - - # Test whether a witness block had the correct effect on the tip - def test_witness_block(self, block, accepted, with_witness=True): - if with_witness: - self.send_message(msg_witness_block(block)) - else: - self.send_message(msg_block(block)) - self.sync_with_ping() - assert_equal(self.connection.rpc.getbestblockhash() == block.hash, accepted) - # Used to keep track of anyone-can-spend outputs that we can use in the tests -class UTXO(object): +class UTXO(): def __init__(self, sha256, n, nValue): self.sha256 = sha256 self.n = n @@ -111,7 +118,8 @@ class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] + # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + self.extra_args = [["-whitelist=127.0.0.1", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] def setup_network(self): self.setup_nodes() @@ -140,7 +148,7 @@ class SegWitTest(BitcoinTestFramework): ''' Individual tests ''' def test_witness_services(self): self.log.info("Verifying NODE_WITNESS service bit") - assert((self.test_node.connection.nServices & NODE_WITNESS) != 0) + assert((self.test_node.nServices & NODE_WITNESS) != 0) # See if sending a regular transaction works, and create a utxo @@ -199,7 +207,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx]) # Sending witness data before activation is not allowed (anti-spam # rule). - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # TODO: fix synchronization so we can test reject reason # Right now, bitcoind delays sending reject messages for blocks # until the future, making synchronization here difficult. @@ -226,7 +234,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptPubKey)) tx2.rehash() - self.test_node.test_transaction_acceptance(tx2, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, False, True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -243,18 +251,18 @@ class SegWitTest(BitcoinTestFramework): tx3.rehash() # Note that this should be rejected for the premature witness reason, # rather than a policy check, since segwit hasn't activated yet. - self.std_node.test_transaction_acceptance(tx3, True, False, b'no-witness-yet') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'no-witness-yet') # If we send without witness, it should be accepted. - self.std_node.test_transaction_acceptance(tx3, False, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, False, True) # Now create a new anyone-can-spend utxo for the next test. tx4 = CTransaction() tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), CScript([p2sh_program]))) tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, CScript([OP_TRUE]))) tx4.rehash() - self.test_node.test_transaction_acceptance(tx3, False, True) - self.test_node.test_transaction_acceptance(tx4, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, False, True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -315,7 +323,7 @@ class SegWitTest(BitcoinTestFramework): assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) # This empty block should be valid. - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Try to tweak the nonce block_2 = self.build_next_block() @@ -326,7 +334,7 @@ class SegWitTest(BitcoinTestFramework): assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) # This should also be valid. - self.test_node.test_witness_block(block_2, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=True) # Now test commitments with actual transactions assert (len(self.utxo) > 0) @@ -359,7 +367,7 @@ class SegWitTest(BitcoinTestFramework): block_3.rehash() block_3.solve() - self.test_node.test_witness_block(block_3, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) # Add a different commitment with different nonce, but in the # right location, and with some funds burned(!). @@ -373,7 +381,7 @@ class SegWitTest(BitcoinTestFramework): block_3.rehash() assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns block_3.solve() - self.test_node.test_witness_block(block_3, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=True) # Finally test that a block with no witness transactions can # omit the commitment. @@ -385,7 +393,7 @@ class SegWitTest(BitcoinTestFramework): block_4.vtx.append(tx3) block_4.hashMerkleRoot = block_4.calc_merkle_root() block_4.solve() - self.test_node.test_witness_block(block_4, with_witness=False, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, with_witness=False, accepted=True) # Update available utxo's for use in later test. self.utxo.pop(0) @@ -426,11 +434,11 @@ class SegWitTest(BitcoinTestFramework): # Change the nonce -- should not cause the block to be permanently # failed block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(1) ] - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Changing the witness nonce doesn't change the block hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(0) ] - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) def test_witness_block_size(self): @@ -495,7 +503,7 @@ class SegWitTest(BitcoinTestFramework): # limit assert(len(block.serialize(True)) > 2*1024*1024) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now resize the second transaction to make the block fit. cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) @@ -505,7 +513,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update available utxo's self.utxo.pop(0) @@ -572,7 +580,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx]) # Extra witness data should not be allowed. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Try extra signature data. Ok if we're not spending a witness output. block.vtx[1].wit.vtxinwit = [] @@ -581,7 +589,7 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try extra witness/signature data on an input that DOES require a # witness @@ -597,7 +605,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx2]) # This has extra witness data, so it should fail. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now get rid of the extra witness, but add extra scriptSig data tx2.vin[0].scriptSig = CScript([OP_TRUE]) @@ -609,7 +617,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # This has extra signature data for a witness input, so it should fail. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now get rid of the extra scriptsig on the witness input, and verify # success (even with extra scriptsig data in the non-witness input) @@ -618,7 +626,7 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update utxo for later tests self.utxo.pop(0) @@ -651,14 +659,14 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now reduce the length of the stack element tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(MAX_SCRIPT_ELEMENT_SIZE) add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update the utxo for later tests self.utxo.pop() @@ -693,7 +701,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Try again with one less byte in the witness program witness_program = CScript([b'a'*520]*19 + [OP_DROP]*62 + [OP_TRUE]) @@ -708,7 +716,7 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -734,7 +742,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Try various ways to spend tx that should all break. # This "broken" transaction serializer will not normalize @@ -769,7 +777,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now try using a too short vtxinwit tx2.wit.vtxinwit.pop() @@ -777,7 +785,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now make one of the intermediate witnesses be incorrect tx2.wit.vtxinwit.append(CTxInWitness()) @@ -786,13 +794,13 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Fix the broken witness and the block should be accepted. tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_program] block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -832,11 +840,11 @@ class SegWitTest(BitcoinTestFramework): # its from) assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(len(self.nodes[1].getrawmempool()), 0) - self.old_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.old_node, tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=False) # But eliminating the witness should fix it - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) # Cleanup: mine the first transaction and update utxo self.nodes[0].generate(1) @@ -868,11 +876,11 @@ class SegWitTest(BitcoinTestFramework): # Verify that unnecessary witnesses are rejected. self.test_node.announce_tx_and_wait_for_getdata(tx) assert_equal(len(self.nodes[0].getrawmempool()), 0) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=False) # Verify that removing the witness succeeds. self.test_node.announce_tx_and_wait_for_getdata(tx) - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) # Now try to add extra witness data to a valid witness tx. witness_program = CScript([OP_TRUE]) @@ -897,24 +905,24 @@ class SegWitTest(BitcoinTestFramework): # Node will not be blinded to the transaction self.std_node.announce_tx_and_wait_for_getdata(tx3) - self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'tx-size') self.std_node.announce_tx_and_wait_for_getdata(tx3) - self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'tx-size') # Remove witness stuffing, instead add extra witness push on stack tx3.vout[0] = CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE])) tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program ] tx3.rehash() - self.test_node.test_transaction_acceptance(tx2, with_witness=True, accepted=True) - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) # Get rid of the extra witness, and verify acceptance. tx3.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) # Test that getrawtransaction returns correct witness information @@ -953,20 +961,20 @@ class SegWitTest(BitcoinTestFramework): self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block1, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block1, True) block2 = self.build_next_block(nVersion=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block2, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, True) block3 = self.build_next_block(nVersion=(VB_TOP_BITS | (1<<15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block3, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, # and the right thing happens. @@ -996,7 +1004,7 @@ class SegWitTest(BitcoinTestFramework): # This gives us a witness commitment. assert(len(block.vtx[0].wit.vtxinwit) == 1) assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) non_wit_block = self.test_node.request_block(block.sha256, 2) @@ -1050,7 +1058,7 @@ class SegWitTest(BitcoinTestFramework): p2sh_tx.rehash() # Mine it on test_node to create the confirmed output. - self.test_node.test_transaction_acceptance(p2sh_tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_tx, with_witness=True, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1062,7 +1070,7 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(8000, scriptPubKey)) # Might burn this later tx.rehash() - self.std_node.test_transaction_acceptance(tx, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx, with_witness=True, accepted=segwit_activated) # Now create something that looks like a P2PKH output. This won't be spendable. scriptPubKey = CScript([OP_0, hash160(witness_hash)]) @@ -1079,7 +1087,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, scriptPubKey)] tx2.rehash() - self.std_node.test_transaction_acceptance(tx2, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=segwit_activated) # Now update self.utxo for later tests. tx3 = CTransaction() @@ -1092,13 +1100,13 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit.append(CTxInWitness()) tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] tx3.rehash() - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) else: # tx and tx2 didn't go anywhere; just clean up the p2sh_tx output. tx3.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] tx3.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, witness_program)] tx3.rehash() - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1122,7 +1130,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop(0) for i in range(NUM_TESTS): self.utxo.append(UTXO(tx.sha256, i, split_value)) @@ -1141,8 +1149,8 @@ class SegWitTest(BitcoinTestFramework): tx.vin = [CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")] tx.vout = [CTxOut(self.utxo[0].nValue-1000, scriptPubKey)] tx.rehash() - self.std_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) self.utxo.pop(0) temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) @@ -1161,8 +1169,8 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() # Gets accepted to test_node, because standardness of outputs isn't # checked with fRequireStandard - self.test_node.test_transaction_acceptance(tx2, with_witness=True, accepted=True) - self.std_node.test_transaction_acceptance(tx2, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=False) temp_utxo.pop() # last entry in temp_utxo was the output we just spent temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -1178,7 +1186,7 @@ class SegWitTest(BitcoinTestFramework): tx3.rehash() # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) self.test_node.sync_with_ping() with mininode_lock: assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) @@ -1186,7 +1194,7 @@ class SegWitTest(BitcoinTestFramework): # Building a block with the transaction must be valid, however. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2, tx3]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) sync_blocks(self.nodes) # Add utxo to our list @@ -1204,7 +1212,7 @@ class SegWitTest(BitcoinTestFramework): # This next line will rehash the coinbase and update the merkle # root, and solve. self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) spend_tx = CTransaction() spend_tx.vin = [CTxIn(COutPoint(block.vtx[0].sha256, 0), b"")] @@ -1218,13 +1226,13 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - self.test_node.test_witness_block(block2, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=False) # Advancing one more block should allow the spend. self.nodes[0].generate(1) block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - self.test_node.test_witness_block(block2, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=True) sync_blocks(self.nodes) @@ -1245,11 +1253,11 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) tx.rehash() - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) # Mine this transaction in preparation for following tests. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) sync_blocks(self.nodes) self.utxo.pop(0) @@ -1266,19 +1274,19 @@ class SegWitTest(BitcoinTestFramework): # Too-large input value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue+1, key) self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Too-small input value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue-1, key) block.vtx.pop() # remove last tx self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now try correct value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue, key) block.vtx.pop() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) @@ -1302,7 +1310,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) block = self.build_next_block() used_sighash_single_out_of_bounds = False @@ -1344,7 +1352,7 @@ class SegWitTest(BitcoinTestFramework): # Test the block periodically, if we're close to maxblocksize if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) block = self.build_next_block() if (not used_sighash_single_out_of_bounds): @@ -1352,7 +1360,7 @@ class SegWitTest(BitcoinTestFramework): # Test the transactions we've added to the block if (len(block.vtx) > 1): self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now test witness version 0 P2PKH transactions pubkeyhash = hash160(pubkey) @@ -1374,7 +1382,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin[0].scriptSig = CScript([signature, pubkey]) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Move the signature to the witness. block.vtx.pop() @@ -1384,7 +1392,7 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) temp_utxos.pop(0) @@ -1403,7 +1411,7 @@ class SegWitTest(BitcoinTestFramework): index += 1 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) for i in range(len(tx.vout)): self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) @@ -1430,10 +1438,10 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() # Verify mempool acceptance and block validity - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True, with_witness=segwit_activated) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=segwit_activated) sync_blocks(self.nodes) # Now test attempts to spend the output. @@ -1447,12 +1455,12 @@ class SegWitTest(BitcoinTestFramework): # will require a witness to spend a witness program regardless of # segwit activation. Note that older bitcoind's that are not # segwit-aware would also reject this for failing CLEANSTACK. - self.test_node.test_transaction_acceptance(spend_tx, with_witness=False, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) # Try to put the witness script in the scriptSig, should also fail. spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) spend_tx.rehash() - self.test_node.test_transaction_acceptance(spend_tx, with_witness=False, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) # Now put the witness script in the witness, should succeed after # segwit activates. @@ -1462,7 +1470,7 @@ class SegWitTest(BitcoinTestFramework): spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ b'a', witness_program ] # Verify mempool acceptance - self.test_node.test_transaction_acceptance(spend_tx, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=True, accepted=segwit_activated) block = self.build_next_block() self.update_witness_block_with_transactions(block, [spend_tx]) @@ -1470,9 +1478,9 @@ class SegWitTest(BitcoinTestFramework): # should be valid. If we're after activation, then sending this with # witnesses should be valid. if segwit_activated: - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) else: - self.test_node.test_witness_block(block, accepted=True, with_witness=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=False) # Update self.utxo self.utxo.pop(0) @@ -1493,7 +1501,7 @@ class SegWitTest(BitcoinTestFramework): # Restart with the new binary self.stop_node(node_id) - self.start_node(node_id, extra_args=[]) + self.start_node(node_id, extra_args=["-vbparams=segwit:0:999999999999"]) connect_nodes(self.nodes[0], node_id) sync_blocks(self.nodes) @@ -1556,7 +1564,7 @@ class SegWitTest(BitcoinTestFramework): block_1 = self.build_next_block() self.update_witness_block_with_transactions(block_1, [tx]) - self.test_node.test_witness_block(block_1, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_1, accepted=True) tx2 = CTransaction() # If we try to spend the first n-1 outputs from tx, that should be @@ -1573,7 +1581,7 @@ class SegWitTest(BitcoinTestFramework): block_2 = self.build_next_block() self.update_witness_block_with_transactions(block_2, [tx2]) - self.test_node.test_witness_block(block_2, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=False) # Try dropping the last input in tx2, and add an output that has # too many sigops (contributing to legacy sigop count). @@ -1586,14 +1594,14 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() block_3 = self.build_next_block() self.update_witness_block_with_transactions(block_3, [tx2]) - self.test_node.test_witness_block(block_3, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) # If we drop the last checksig in this output, the tx should succeed. block_4 = self.build_next_block() tx2.vout[-1].scriptPubKey = CScript([OP_CHECKSIG]*(checksig_count-1)) tx2.rehash() self.update_witness_block_with_transactions(block_4, [tx2]) - self.test_node.test_witness_block(block_4, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, accepted=True) # Reset the tip back down for the next test sync_blocks(self.nodes) @@ -1609,7 +1617,7 @@ class SegWitTest(BitcoinTestFramework): tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_justright ] tx2.rehash() self.update_witness_block_with_transactions(block_5, [tx2]) - self.test_node.test_witness_block(block_5, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_5, accepted=True) # TODO: test p2sh sigop counting @@ -1687,7 +1695,7 @@ class SegWitTest(BitcoinTestFramework): # Confirm it in a block. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try to spend it. Send it to a P2WSH output, which we'll # use in the next test. @@ -1706,11 +1714,11 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() # Should fail policy test. - self.test_node.test_transaction_acceptance(tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') # But passes consensus. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 2: P2WSH # Try to spend the P2WSH output created in last test. @@ -1726,11 +1734,11 @@ class SegWitTest(BitcoinTestFramework): sign_P2PK_witness_input(witness_program, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) # Should fail policy test. - self.test_node.test_transaction_acceptance(tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') # But passes consensus. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx3]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 3: P2SH(P2WSH) # Try to spend the P2SH output created in the last test. @@ -1743,10 +1751,10 @@ class SegWitTest(BitcoinTestFramework): sign_P2PK_witness_input(witness_program, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) # Should fail policy test. - self.test_node.test_transaction_acceptance(tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx4]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 4: Uncompressed pubkeys should still be valid in non-segwit # transactions. @@ -1758,10 +1766,10 @@ class SegWitTest(BitcoinTestFramework): tx5.vin[0].scriptSig = CScript([signature, pubkey]) tx5.rehash() # Should pass policy and consensus. - self.test_node.test_transaction_acceptance(tx5, True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx5, True, True) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx5]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) def test_non_standard_witness(self): @@ -1791,7 +1799,7 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) tx.rehash() txid = tx.sha256 - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1816,45 +1824,45 @@ class SegWitTest(BitcoinTestFramework): # Testing native P2WSH # Witness stack size, excluding witnessScript, over 100 is non-standard p2wsh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - self.std_node.test_transaction_acceptance(p2wsh_txs[0], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[0], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[0], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[0], True, True) # Stack element size over 80 bytes is non-standard p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2wsh_txs[1], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[1], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[1], True, True) # Standard nodes should accept if element size is not over 80 bytes p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2wsh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, True) # witnessScript size at 3600 bytes is standard p2wsh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - self.test_node.test_transaction_acceptance(p2wsh_txs[2], True, True) - self.std_node.test_transaction_acceptance(p2wsh_txs[2], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[2], True, True) # witnessScript size at 3601 bytes is non-standard p2wsh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - self.std_node.test_transaction_acceptance(p2wsh_txs[3], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[3], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[3], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[3], True, True) # Repeating the same tests with P2SH-P2WSH p2sh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - self.std_node.test_transaction_acceptance(p2sh_txs[0], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[0], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[0], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[0], True, True) p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2sh_txs[1], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[1], True, True) p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2sh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, True) p2sh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - self.test_node.test_transaction_acceptance(p2sh_txs[2], True, True) - self.std_node.test_transaction_acceptance(p2sh_txs[2], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[2], True, True) p2sh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - self.std_node.test_transaction_acceptance(p2sh_txs[3], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[3], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[3], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[3], True, True) self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node # Valid but non-standard transactions in a block should be accepted by standard node @@ -1867,21 +1875,14 @@ class SegWitTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.test_node = TestNode() # sets NODE_WITNESS|NODE_NETWORK - self.old_node = TestNode() # only NODE_NETWORK - self.std_node = TestNode() # for testing node1 (fRequireStandard=true) - - self.p2p_connections = [self.test_node, self.old_node] - - self.connections = [] - self.connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node, services=NODE_NETWORK|NODE_WITNESS)) - self.connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.old_node, services=NODE_NETWORK)) - self.connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], self.std_node, services=NODE_NETWORK|NODE_WITNESS)) - self.test_node.add_connection(self.connections[0]) - self.old_node.add_connection(self.connections[1]) - self.std_node.add_connection(self.connections[2]) - - NetworkThread().start() # Start up network handling in another thread + # self.test_node sets NODE_WITNESS|NODE_NETWORK + self.test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + # self.old_node sets only NODE_NETWORK + self.old_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK) + # self.std_node is for testing node1 (fRequireStandard=true) + self.std_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + + network_thread_start() # Keep a place to store utxo's that can be used in later tests self.utxo = [] diff --git a/test/functional/p2p-timeouts.py b/test/functional/p2p-timeouts.py index 51d4769efc..984a3c8b90 100755 --- a/test/functional/p2p-timeouts.py +++ b/test/functional/p2p-timeouts.py @@ -27,8 +27,8 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(NodeConnCB): - def on_version(self, conn, message): +class TestNode(P2PInterface): + def on_version(self, message): # Don't send a verack in response pass @@ -39,46 +39,37 @@ class TimeoutsTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.no_verack_node = TestNode() # never send verack - self.no_version_node = TestNode() # never send version (just ping) - self.no_send_node = TestNode() # never send anything + no_verack_node = self.nodes[0].add_p2p_connection(TestNode()) + no_version_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) + no_send_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_verack_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_version_node, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_send_node, send_version=False)) - self.no_verack_node.add_connection(connections[0]) - self.no_version_node.add_connection(connections[1]) - self.no_send_node.add_connection(connections[2]) - - NetworkThread().start() # Start up network handling in another thread + network_thread_start() sleep(1) - assert(self.no_verack_node.connected) - assert(self.no_version_node.connected) - assert(self.no_send_node.connected) + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected - ping_msg = msg_ping() - connections[0].send_message(ping_msg) - connections[1].send_message(ping_msg) + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) sleep(30) - assert "version" in self.no_verack_node.last_message + assert "version" in no_verack_node.last_message - assert(self.no_verack_node.connected) - assert(self.no_version_node.connected) - assert(self.no_send_node.connected) + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected - connections[0].send_message(ping_msg) - connections[1].send_message(ping_msg) + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) sleep(31) - assert(not self.no_verack_node.connected) - assert(not self.no_version_node.connected) - assert(not self.no_send_node.connected) + assert not no_verack_node.connected + assert not no_version_node.connected + assert not no_send_node.connected if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/p2p-versionbits-warning.py b/test/functional/p2p-versionbits-warning.py index f9bef2580a..d29d43ebed 100755 --- a/test/functional/p2p-versionbits-warning.py +++ b/test/functional/p2p-versionbits-warning.py @@ -23,8 +23,8 @@ WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible un WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("^Warning.*versionbit") -class TestNode(NodeConnCB): - def on_inv(self, conn, message): +class TestNode(P2PInterface): + def on_inv(self, message): pass class VersionBitsWarningTest(BitcoinTestFramework): @@ -64,16 +64,12 @@ class VersionBitsWarningTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connection and start up the network thread. - test_node = TestNode() + self.nodes[0].add_p2p_connection(TestNode()) - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - test_node.add_connection(connections[0]) - - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here - test_node.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() # 1. Have the node mine one period worth of blocks self.nodes[0].generate(VB_PERIOD) @@ -81,7 +77,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): # 2. Now build one period of blocks on the tip, with < VB_THRESHOLD # blocks signaling some unknown bit. nVersion = VB_TOP_BITS | (1<<VB_UNKNOWN_BIT) - self.send_blocks_with_version(test_node, VB_THRESHOLD-1, nVersion) + self.send_blocks_with_version(self.nodes[0].p2p, VB_THRESHOLD-1, nVersion) # Fill rest of period with regular version blocks self.nodes[0].generate(VB_PERIOD - VB_THRESHOLD + 1) @@ -92,7 +88,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): # 3. Now build one period of blocks with >= VB_THRESHOLD blocks signaling # some unknown bit - self.send_blocks_with_version(test_node, VB_THRESHOLD, nVersion) + self.send_blocks_with_version(self.nodes[0].p2p, VB_THRESHOLD, nVersion) self.nodes[0].generate(VB_PERIOD - VB_THRESHOLD) # Might not get a versionbits-related alert yet, as we should # have gotten a different alert due to more than 51/100 blocks diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index 2777cb9693..aa519eb8e8 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -15,6 +15,25 @@ Test the following RPCs: from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * + +class multidict(dict): + """Dictionary that allows duplicate keys. + + Constructed with a list of (key, value) tuples. When dumped by the json module, + will output invalid json with repeated keys, eg: + >>> json.dumps(multidict([(1,2),(1,2)]) + '{"1": 2, "1": 2}' + + Used to test calls to rpc methods with repeated keys in the json object.""" + + def __init__(self, x): + dict.__init__(self, x) + self.x = x + + def items(self): + return self.x + + # Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -39,6 +58,41 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(5) self.sync_all() + # Test `createrawtransaction` required parameters + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction) + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, []) + + # Test `createrawtransaction` invalid extra parameters + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [], {}, 0, False, 'foo') + + # Test `createrawtransaction` invalid `inputs` + txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000' + assert_raises_rpc_error(-3, "Expected type array", self.nodes[0].createrawtransaction, 'foo', {}) + assert_raises_rpc_error(-1, "JSON value is not an object as expected", self.nodes[0].createrawtransaction, ['foo'], {}) + assert_raises_rpc_error(-8, "txid must be hexadecimal string", self.nodes[0].createrawtransaction, [{}], {}) + assert_raises_rpc_error(-8, "txid must be hexadecimal string", self.nodes[0].createrawtransaction, [{'txid': 'foo'}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 'foo'}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, vout must be positive", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': -1}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, sequence number is out of range", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 0, 'sequence': -1}], {}) + + # Test `createrawtransaction` invalid `outputs` + address = self.nodes[0].getnewaddress() + assert_raises_rpc_error(-3, "Expected type object", self.nodes[0].createrawtransaction, [], 'foo') + assert_raises_rpc_error(-8, "Data must be hexadecimal string", self.nodes[0].createrawtransaction, [], {'data': 'foo'}) + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].createrawtransaction, [], {'foo': 0}) + assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].createrawtransaction, [], {address: 'foo'}) + assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].createrawtransaction, [], {address: -1}) + assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], multidict([(address, 1), (address, 1)])) + + # Test `createrawtransaction` invalid `locktime` + assert_raises_rpc_error(-3, "Expected type number", self.nodes[0].createrawtransaction, [], {}, 'foo') + assert_raises_rpc_error(-8, "Invalid parameter, locktime out of range", self.nodes[0].createrawtransaction, [], {}, -1) + assert_raises_rpc_error(-8, "Invalid parameter, locktime out of range", self.nodes[0].createrawtransaction, [], {}, 4294967296) + + # Test `createrawtransaction` invalid `replaceable` + assert_raises_rpc_error(-3, "Expected type bool", self.nodes[0].createrawtransaction, [], {}, 0, 'foo') + ######################################### # sendrawtransaction with missing input # ######################################### @@ -50,6 +104,36 @@ class RawTransactionsTest(BitcoinTestFramework): # This will raise an exception since there are missing inputs assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + ##################################### + # getrawtransaction with block hash # + ##################################### + + # make a tx by sending then generate 2 blocks; block1 has the tx in it + tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + block1, block2 = self.nodes[2].generate(2) + self.sync_all() + # We should be able to get the raw transaction by providing the correct block + gottx = self.nodes[0].getrawtransaction(tx, True, block1) + assert_equal(gottx['txid'], tx) + assert_equal(gottx['in_active_chain'], True) + # We should not have the 'in_active_chain' flag when we don't provide a block + gottx = self.nodes[0].getrawtransaction(tx, True) + assert_equal(gottx['txid'], tx) + assert 'in_active_chain' not in gottx + # We should not get the tx if we provide an unrelated block + assert_raises_rpc_error(-5, "No such transaction found", self.nodes[0].getrawtransaction, tx, True, block2) + # An invalid block hash should raise the correct errors + assert_raises_rpc_error(-8, "parameter 3 must be hexadecimal", self.nodes[0].getrawtransaction, tx, True, True) + assert_raises_rpc_error(-8, "parameter 3 must be hexadecimal", self.nodes[0].getrawtransaction, tx, True, "foobar") + assert_raises_rpc_error(-8, "parameter 3 must be of length 64", self.nodes[0].getrawtransaction, tx, True, "abcd1234") + assert_raises_rpc_error(-5, "Block hash not found", self.nodes[0].getrawtransaction, tx, True, "0000000000000000000000000000000000000000000000000000000000000000") + # Undo the blocks and check in_active_chain + self.nodes[0].invalidateblock(block1) + gottx = self.nodes[0].getrawtransaction(txid=tx, verbose=True, blockhash=block1) + assert_equal(gottx['in_active_chain'], False) + self.nodes[0].reconsiderblock(block1) + assert_equal(self.nodes[0].getbestblockhash(), block2) + ######################### # RAW TX MULTISIG TESTS # ######################### @@ -169,6 +253,17 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + # decoderawtransaction tests + # witness transaction + encrawtx = "010000000001010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f50500000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, True) # decode as witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # force decode as non-witness transaction + # non-witness transaction + encrawtx = "01000000010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f505000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, False) # decode as non-witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + # getrawtransaction tests # 1. valid parameters - only supply txid txHash = rawTx["hash"] @@ -188,13 +283,13 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index db6fc86b82..97da19546f 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -3,97 +3,83 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -def get_sub_array_from_array(object_array, to_match): - ''' - Finds and returns a sub array from an array of arrays. - to_match should be a unique idetifier of a sub array - ''' - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - return item - return [] +from test_framework.util import (assert_array_result, + assert_equal, + assert_raises_rpc_error, + ) class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.enable_mocktime() def run_test(self): - ''' - listreceivedbyaddress Test - ''' + # Generate block to get out of IBD + self.nodes[0].generate(1) + + self.log.info("listreceivedbyaddress Test") + # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check not listed in listreceivedbyaddress because has 0 confirmations + # Check not listed in listreceivedbyaddress because has 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - { }, - True) - #Bury Tx under 10 block so it will be returned by listreceivedbyaddress + {"address": addr}, + {}, + True) + # Bury Tx under 10 block so it will be returned by listreceivedbyaddress self.nodes[1].generate(10) self.sync_all() assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence < 10 + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence < 10 assert_array_result(self.nodes[1].listreceivedbyaddress(5), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence > 10, should not find Tx - assert_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence > 10, should not find Tx + assert_array_result(self.nodes[1].listreceivedbyaddress(11), {"address": addr}, {}, True) - #Empty Tx + # Empty Tx addr = self.nodes[1].getnewaddress() - assert_array_result(self.nodes[1].listreceivedbyaddress(0,True), - {"address":addr}, - {"address":addr, "account":"", "amount":0, "confirmations":0, "txids":[]}) + assert_array_result(self.nodes[1].listreceivedbyaddress(0, True), + {"address": addr}, + {"address": addr, "account": "", "amount": 0, "confirmations": 0, "txids": []}) + + self.log.info("getreceivedbyaddress Test") - ''' - getreceivedbyaddress Test - ''' # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check balance is 0 because of 0 confirmations + # Check balance is 0 because of 0 confirmations balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) - #Check balance is 0.1 - balance = self.nodes[1].getreceivedbyaddress(addr,0) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + # Check balance is 0.1 + balance = self.nodes[1].getreceivedbyaddress(addr, 0) + assert_equal(balance, Decimal("0.1")) - #Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress + # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress self.nodes[1].generate(10) self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.1")) + + # Trying to getreceivedby for an address the wallet doesn't own should return an error + assert_raises_rpc_error(-4, "Address not found in wallet", self.nodes[0].getreceivedbyaddress, addr) + + self.log.info("listreceivedbyaccount + getreceivedbyaccount Test") - ''' - listreceivedbyaccount + getreceivedbyaccount Test - ''' - #set pre-state + # set pre-state addrArr = self.nodes[1].getnewaddress() account = self.nodes[1].getaccount(addrArr) - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(),{"account":account}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount() if r["account"] == account][0] balance_by_account = self.nodes[1].getreceivedbyaccount(account) txid = self.nodes[0].sendtoaddress(addr, 0.1) @@ -101,40 +87,34 @@ class ReceivedByTest(BitcoinTestFramework): # listreceivedbyaccount should return received_by_account_json because of 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - received_by_account_json) + {"account": account}, + received_by_account_json) # getreceivedbyaddress should return same balance because of 0 confirmations balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account: - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account) self.nodes[1].generate(10) self.sync_all() # listreceivedbyaccount should return updated account balance assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) + {"account": account}, + {"account": received_by_account_json["account"], "amount": (received_by_account_json["amount"] + Decimal("0.1"))}) # getreceivedbyaddress should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account + Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account + Decimal("0.1")) - #Create a new account named "mynewaccount" that has a 0 balance + # Create a new account named "mynewaccount" that has a 0 balance self.nodes[1].getaccountaddress("mynewaccount") - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(0,True),{"account":"mynewaccount"}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount(0, True) if r["account"] == "mynewaccount"][0] # Test includeempty of listreceivedbyaccount - if received_by_account_json["amount"] != Decimal("0.0"): - raise AssertionError("Wrong balance returned by listreceivedbyaccount, %0.2f"%(received_by_account_json["amount"])) + assert_equal(received_by_account_json["amount"], Decimal("0.0")) # Test getreceivedbyaccount for 0 amount accounts balance = self.nodes[1].getreceivedbyaccount("mynewaccount") - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) if __name__ == '__main__': ReceivedByTest().main() diff --git a/test/functional/replace-by-fee.py b/test/functional/replace-by-fee.py index 269d57775c..815e964848 100755 --- a/test/functional/replace-by-fee.py +++ b/test/functional/replace-by-fee.py @@ -72,8 +72,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): ["-mempoolreplacement=0"]] def run_test(self): + # Leave IBD + self.nodes[0].generate(1) + make_utxo(self.nodes[0], 1*COIN) + # Ensure nodes are synced + self.sync_all() + self.log.info("Running test simple doublespend...") self.test_simple_doublespend() @@ -110,13 +116,18 @@ class ReplaceByFeeTest(BitcoinTestFramework): """Simple doublespend""" tx0_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) + # make_utxo may have generated a bunch of blocks, so we need to sync + # before we can spend the coins generated, or else the resulting + # transactions might not be accepted by our peers. + self.sync_all() + tx1a = CTransaction() tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1*COIN, CScript([b'a']))] tx1a_hex = txToHex(tx1a) tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) - self.sync_all([self.nodes]) + self.sync_all() # Should fail because we haven't changed the fee tx1b = CTransaction() diff --git a/test/functional/segwit.py b/test/functional/segwit.py index 6ecade7cb6..338fa1bc52 100755 --- a/test/functional/segwit.py +++ b/test/functional/segwit.py @@ -77,9 +77,10 @@ class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0"], - ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"], - ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]] + # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0", "-vbparams=segwit:0:999999999999"], + ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999"], + ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-vbparams=segwit:0:999999999999"]] def setup_network(self): super().setup_network() diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py index fe577dc20a..256227f721 100755 --- a/test/functional/sendheaders.py +++ b/test/functional/sendheaders.py @@ -4,11 +4,24 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. -Setup: +Setup: -- Two nodes, two p2p connections to node0. One p2p connection should only ever - receive inv's (omitted from testing description below, this is our control). - Second node is used for creating reorgs. +- Two nodes: + - node0 is the node-under-test. We create two p2p connections to it. The + first p2p connection is a control and should only ever receive inv's. The + second p2p connection tests the headers sending logic. + - node1 is used to create reorgs. + +test_null_locators +================== + +Sends two getheaders requests with null locator values. First request's hashstop +value refers to validated block, while second request's hashstop value refers to +a block which hasn't been validated. Verifies only the first request returns +headers. + +test_nonnull_locators +===================== Part 1: No headers announcements before "sendheaders" a. node mines a block [expect: inv] @@ -72,159 +85,197 @@ d. Announce 49 headers that don't connect. e. Announce one more that doesn't connect. Expect: disconnect. """ - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * from test_framework.blocktools import create_block, create_coinbase +from test_framework.mininode import ( + CBlockHeader, + CInv, + NODE_WITNESS, + network_thread_start, + P2PInterface, + mininode_lock, + msg_block, + msg_getblocks, + msg_getdata, + msg_getheaders, + msg_headers, + msg_inv, + msg_sendheaders, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + sync_blocks, + wait_until, +) +DIRECT_FETCH_RESPONSE_TIME = 0.05 -direct_fetch_response_time = 0.05 - -class TestNode(NodeConnCB): +class BaseNode(P2PInterface): def __init__(self): super().__init__() + self.block_announced = False self.last_blockhash_announced = None - def clear_last_announcement(self): - with mininode_lock: - self.block_announced = False - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - - # Request data for a list of block hashes - def get_data(self, block_hashes): + def send_get_data(self, block_hashes): + """Request data for a list of block hashes.""" msg = msg_getdata() for x in block_hashes: msg.inv.append(CInv(2, x)) - self.connection.send_message(msg) + self.send_message(msg) - def get_headers(self, locator, hashstop): + def send_get_headers(self, locator, hashstop): msg = msg_getheaders() msg.locator.vHave = locator msg.hashstop = hashstop - self.connection.send_message(msg) + self.send_message(msg) def send_block_inv(self, blockhash): msg = msg_inv() msg.inv = [CInv(2, blockhash)] - self.connection.send_message(msg) + self.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) - def on_inv(self, conn, message): + def send_getblocks(self, locator): + getblocks_message = msg_getblocks() + getblocks_message.locator.vHave = locator + self.send_message(getblocks_message) + + def wait_for_getdata(self, hash_list, timeout=60): + if hash_list == []: + return + + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def wait_for_block_announcement(self, block_hash, timeout=60): + test_function = lambda: self.last_blockhash_announced == block_hash + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def on_inv(self, message): self.block_announced = True self.last_blockhash_announced = message.inv[-1].hash - def on_headers(self, conn, message): + def on_headers(self, message): if len(message.headers): self.block_announced = True message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 - # Test whether the last announcement we received had the - # right header or the right inv - # inv and headers should be lists of block hashes + def clear_last_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + def check_last_announcement(self, headers=None, inv=None): - expect_headers = headers if headers != None else [] - expect_inv = inv if inv != None else [] + """Test whether the last announcement received had the right header or the right inv. + + inv and headers should be lists of block hashes.""" + test_function = lambda: self.block_announced wait_until(test_function, timeout=60, lock=mininode_lock) + with mininode_lock: self.block_announced = False - success = True compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] - if compare_inv != expect_inv: - success = False + if inv is not None: + assert_equal(compare_inv, inv) - hash_headers = [] + compare_headers = [] if "headers" in self.last_message: - # treat headers as a list of block hashes - hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ] - if hash_headers != expect_headers: - success = False + compare_headers = [x.sha256 for x in self.last_message["headers"].headers] + if headers is not None: + assert_equal(compare_headers, headers) self.last_message.pop("inv", None) self.last_message.pop("headers", None) - return success - - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def wait_for_block_announcement(self, block_hash, timeout=60): - test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def send_header_for_blocks(self, new_blocks): - headers_message = msg_headers() - headers_message.headers = [ CBlockHeader(b) for b in new_blocks ] - self.send_message(headers_message) - - def send_getblocks(self, locator): - getblocks_message = msg_getblocks() - getblocks_message.locator.vHave = locator - self.send_message(getblocks_message) class SendHeadersTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - # mine count blocks and return the new tip def mine_blocks(self, count): + """Mine count blocks and return the new tip.""" + # Clear out last block announcement from each p2p listener - [ x.clear_last_announcement() for x in self.p2p_connections ] + [x.clear_last_announcement() for x in self.nodes[0].p2ps] self.nodes[0].generate(count) return int(self.nodes[0].getbestblockhash(), 16) - # mine a reorg that invalidates length blocks (replacing them with - # length+1 blocks). - # Note: we clear the state of our p2p connections after the - # to-be-reorged-out blocks are mined, so that we don't break later tests. - # return the list of block hashes newly mined def mine_reorg(self, length): - self.nodes[0].generate(length) # make sure all invalidated blocks are node0's + """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). + + Note: we clear the state of our p2p connections after the + to-be-reorged-out blocks are mined, so that we don't break later tests. + return the list of block hashes newly mined.""" + + self.nodes[0].generate(length) # make sure all invalidated blocks are node0's sync_blocks(self.nodes, wait=0.1) - for x in self.p2p_connections: + for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_last_announcement() tip_height = self.nodes[1].getblockcount() - hash_to_invalidate = self.nodes[1].getblockhash(tip_height-(length-1)) + hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) - all_hashes = self.nodes[1].generate(length+1) # Must be longer than the orig chain + all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): # Setup the p2p connections and start up the network thread. - inv_node = TestNode() - test_node = TestNode() - - self.p2p_connections = [inv_node, test_node] - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], inv_node)) - # Set nServices to 0 for test_node, so no block download will occur outside of - # direct fetching - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node, services=0)) - inv_node.add_connection(connections[0]) - test_node.add_connection(connections[1]) + inv_node = self.nodes[0].add_p2p_connection(BaseNode()) + # Make sure NODE_NETWORK is not set for test_node, so no block download + # will occur outside of direct fetching + test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here inv_node.wait_for_verack() test_node.wait_for_verack() + # Ensure verack's have been processed by our peer + inv_node.sync_with_ping() + test_node.sync_with_ping() + + self.test_null_locators(test_node, inv_node) + self.test_nonnull_locators(test_node, inv_node) + + def test_null_locators(self, test_node, inv_node): + tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0]) + tip_hash = int(tip["hash"], 16) + + inv_node.check_last_announcement(inv=[tip_hash], headers=[]) + test_node.check_last_announcement(inv=[tip_hash], headers=[]) + + self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=tip_hash) + test_node.check_last_announcement(headers=[tip_hash]) + + self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") + block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) + block.solve() + test_node.send_header_for_blocks([block]) + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) + test_node.sync_with_ping() + assert_equal(test_node.block_announced, False) + inv_node.clear_last_announcement() + test_node.send_message(msg_block(block)) + inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[]) + + def test_nonnull_locators(self, test_node, inv_node): tip = int(self.nodes[0].getbestblockhash(), 16) # PART 1 @@ -233,30 +284,30 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(4): old_tip = tip tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) # Try a few different responses; none should affect next announcement if i == 0: # first request the block - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # next try requesting header and block - test_node.get_headers(locator=[old_tip], hashstop=tip) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[old_tip], hashstop=tip) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) - test_node.clear_last_announcement() # since we requested headers... + test_node.clear_last_announcement() # since we requested headers... elif i == 2: # this time announce own block via headers height = self.nodes[0].getblockcount() last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] block_time = last_time + 1 - new_block = create_block(tip, create_coinbase(height+1), block_time) + new_block = create_block(tip, create_coinbase(height + 1), block_time) new_block.solve() test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_message(msg_block(new_block)) - test_node.sync_with_ping() # make sure this block is processed + test_node.sync_with_ping() # make sure this block is processed inv_node.clear_last_announcement() test_node.clear_last_announcement() @@ -267,15 +318,15 @@ class SendHeadersTest(BitcoinTestFramework): # commence and keep working. test_node.send_message(msg_sendheaders()) prev_tip = int(self.nodes[0].getbestblockhash(), 16) - test_node.get_headers(locator=[prev_tip], hashstop=0) + test_node.send_get_headers(locator=[prev_tip], hashstop=0) test_node.sync_with_ping() # Now that we've synced headers, headers announcements should work tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) - height = self.nodes[0].getblockcount()+1 + height = self.nodes[0].getblockcount() + 1 block_time += 10 # Advance far enough ahead for i in range(10): # Mine i blocks, and alternate announcing either via @@ -284,7 +335,7 @@ class SendHeadersTest(BitcoinTestFramework): # with block header, even though the blocks are never requested for j in range(2): blocks = [] - for b in range(i+1): + for b in range(i + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -298,7 +349,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements - [ inv_node.send_block_inv(x.sha256) for x in blocks ] + [inv_node.send_block_inv(x.sha256) for x in blocks] test_node.wait_for_getdata([x.sha256 for x in blocks]) inv_node.sync_with_ping() else: @@ -309,7 +360,7 @@ class SendHeadersTest(BitcoinTestFramework): # getdata requests (the check is further down) inv_node.send_header_for_blocks(blocks) inv_node.sync_with_ping() - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() inv_node.sync_with_ping() # This block should not be announced to the inv node (since it also @@ -317,8 +368,8 @@ class SendHeadersTest(BitcoinTestFramework): assert "inv" not in inv_node.last_message assert "headers" not in inv_node.last_message tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) height += 1 block_time += 1 @@ -332,16 +383,16 @@ class SendHeadersTest(BitcoinTestFramework): # First try mining a reorg that can propagate with header announcement new_block_hashes = self.mine_reorg(length=7) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=new_block_hashes), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=new_block_hashes) - block_time += 8 + block_time += 8 # Mine a too-large reorg, which should be announced with a single inv new_block_hashes = self.mine_reorg(length=8) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) block_time += 9 @@ -349,42 +400,42 @@ class SendHeadersTest(BitcoinTestFramework): fork_point = int(fork_point, 16) # Use getblocks/getdata - test_node.send_getblocks(locator = [fork_point]) - assert_equal(test_node.check_last_announcement(inv=new_block_hashes), True) - test_node.get_data(new_block_hashes) + test_node.send_getblocks(locator=[fork_point]) + test_node.check_last_announcement(inv=new_block_hashes, headers=[]) + test_node.send_get_data(new_block_hashes) test_node.wait_for_block(new_block_hashes[-1]) for i in range(3): # Mine another block, still should get only an inv tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) if i == 0: # Just get the data -- shouldn't cause headers announcements to resume - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # Send a getheaders message that shouldn't trigger headers announcements # to resume (best header sent will be too old) - test_node.get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 2: - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) # This time, try sending either a getheaders to trigger resumption - # of headers announcements, or mine a new block and inv it, also + # of headers announcements, or mine a new block and inv it, also # triggering resumption of headers announcements. if j == 0: - test_node.get_headers(locator=[tip], hashstop=0) + test_node.send_get_headers(locator=[tip], hashstop=0) test_node.sync_with_ping() else: test_node.send_block_inv(tip) test_node.sync_with_ping() # New blocks should now be announced with header tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) self.log.info("Part 3: success!") @@ -404,7 +455,7 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 inv_node.send_message(msg_block(blocks[-1])) - inv_node.sync_with_ping() # Make sure blocks are processed + inv_node.sync_with_ping() # Make sure blocks are processed test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() @@ -423,9 +474,9 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() @@ -454,13 +505,13 @@ class SendHeadersTest(BitcoinTestFramework): # both blocks (same work as tip) test_node.send_header_for_blocks(blocks[1:2]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 16 more headers should trigger direct fetch for 14 more # blocks test_node.send_header_for_blocks(blocks[2:18]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 1 more header should not trigger any response test_node.last_message.pop("getdata", None) @@ -472,7 +523,7 @@ class SendHeadersTest(BitcoinTestFramework): self.log.info("Part 4: success!") # Now deliver all those blocks we announced. - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent @@ -494,7 +545,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_getheaders() test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) @@ -502,7 +553,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. MAX_UNCONNECTING_HEADERS = 10 - for j in range(MAX_UNCONNECTING_HEADERS+1): + for j in range(MAX_UNCONNECTING_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -524,11 +575,11 @@ class SendHeadersTest(BitcoinTestFramework): # Now try to see how many unconnecting headers we can send # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS - for i in range(5*MAX_UNCONNECTING_HEADERS - 1): + for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[i%len(blocks)]]) + test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() # Eventually this stops working. diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index b3671cbdc5..bd3a3b3fab 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -33,24 +33,17 @@ ServiceProxy class: - uses standard Python json lib """ -try: - import http.client as httplib -except ImportError: - import httplib import base64 import decimal +import http.client import json import logging import socket import time -try: - import urllib.parse as urlparse -except ImportError: - import urlparse - -USER_AGENT = "AuthServiceProxy/0.1" +import urllib.parse HTTP_TIMEOUT = 30 +USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") @@ -60,7 +53,7 @@ class JSONRPCException(Exception): errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' - Exception.__init__(self, errmsg) + super().__init__(errmsg) self.error = rpc_error @@ -69,28 +62,18 @@ def EncodeDecimal(o): return str(o) raise TypeError(repr(o) + " is not JSON serializable") -class AuthServiceProxy(object): +class AuthServiceProxy(): __id_count = 0 # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True): self.__service_url = service_url self._service_name = service_name - self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests - self.__url = urlparse.urlparse(service_url) - if self.__url.port is None: - port = 80 - else: - port = self.__url.port - (user, passwd) = (self.__url.username, self.__url.password) - try: - user = user.encode('utf8') - except AttributeError: - pass - try: - passwd = passwd.encode('utf8') - except AttributeError: - pass + self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests + self.__url = urllib.parse.urlparse(service_url) + port = 80 if self.__url.port is None else self.__url.port + user = None if self.__url.username is None else self.__url.username.encode('utf8') + passwd = None if self.__url.password is None else self.__url.password.encode('utf8') authpair = user + b':' + passwd self.__auth_header = b'Basic ' + base64.b64encode(authpair) @@ -98,11 +81,9 @@ class AuthServiceProxy(object): # Callables re-use the connection of the original proxy self.__conn = connection elif self.__url.scheme == 'https': - self.__conn = httplib.HTTPSConnection(self.__url.hostname, port, - timeout=timeout) + self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=timeout) else: - self.__conn = httplib.HTTPConnection(self.__url.hostname, port, - timeout=timeout) + self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=timeout) def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): @@ -124,31 +105,34 @@ class AuthServiceProxy(object): try: self.__conn.request(method, path, postdata, headers) return self._get_response() - except httplib.BadStatusLine as e: - if e.line == "''": # if connection was closed, try again + except http.client.BadStatusLine as e: + if e.line == "''": # if connection was closed, try again self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() else: raise - except (BrokenPipeError,ConnectionResetError): + except (BrokenPipeError, ConnectionResetError): # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset # ConnectionResetError happens on FreeBSD with Python 3.4 self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() - def __call__(self, *args, **argsn): + def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("-%s-> %s %s" % (AuthServiceProxy.__id_count, self._service_name, + json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) if args and argsn: raise ValueError('Cannot handle both named and positional arguments') - postdata = json.dumps({'version': '1.1', - 'method': self._service_name, - 'params': args or argsn, - 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + return {'version': '1.1', + 'method': self._service_name, + 'params': args or argsn, + 'id': AuthServiceProxy.__id_count} + + def __call__(self, *args, **argsn): + postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) response = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: raise JSONRPCException(response['error']) @@ -158,9 +142,9 @@ class AuthServiceProxy(object): else: return response['result'] - def _batch(self, rpc_call_list): + def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - log.debug("--> "+postdata) + log.debug("--> " + postdata) return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): @@ -187,9 +171,9 @@ class AuthServiceProxy(object): response = json.loads(responsedata, parse_float=decimal.Decimal) elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: - log.debug("<-%s- [%.6f] %s"%(response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: - log.debug("<-- [%.6f] %s"%(elapsed,responsedata)) + log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) return response def __truediv__(self, relative_uri): diff --git a/test/functional/test_framework/blockstore.py b/test/functional/test_framework/blockstore.py index 4b2170a03f..051c57a6c7 100644 --- a/test/functional/test_framework/blockstore.py +++ b/test/functional/test_framework/blockstore.py @@ -10,7 +10,7 @@ import dbm.dumb as dbmd logger = logging.getLogger("TestFramework.blockstore") -class BlockStore(object): +class BlockStore(): """BlockStore helper class. BlockStore keeps a map of blocks and implements helper functions for @@ -100,7 +100,7 @@ class BlockStore(object): def get_blocks(self, inv): responses = [] for i in inv: - if (i.type == 2): # MSG_BLOCK + if (i.type == 2 or i.type == (2 | (1 << 30))): # MSG_BLOCK or MSG_WITNESS_BLOCK data = self.get(i.hash) if data is not None: # Use msg_generic to avoid re-serialization @@ -127,7 +127,7 @@ class BlockStore(object): locator.vHave = r return locator -class TxStore(object): +class TxStore(): def __init__(self, datadir): self.txDB = dbmd.open(datadir + "/transactions", 'c') @@ -153,7 +153,7 @@ class TxStore(object): def get_transactions(self, inv): responses = [] for i in inv: - if (i.type == 1): # MSG_TX + if (i.type == 1 or i.type == (1 | (1 << 30))): # MSG_TX or MSG_WITNESS_TX tx = self.get(i.hash) if tx is not None: responses.append(msg_generic(b"tx", tx)) diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py index bfbc0c3b03..f0f5c847ca 100755 --- a/test/functional/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -27,7 +27,7 @@ logger=logging.getLogger("TestFramework.comptool") global mininode_lock -class RejectResult(object): +class RejectResult(): """Outcome that expects rejection of a transaction or block.""" def __init__(self, code, reason=b''): self.code = code @@ -39,11 +39,10 @@ class RejectResult(object): def __repr__(self): return '%i:%s' % (self.code,self.reason or '*') -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self, block_store, tx_store): super().__init__() - self.conn = None self.bestblockhash = None self.block_store = block_store self.block_request_map = {} @@ -58,43 +57,40 @@ class TestNode(NodeConnCB): self.lastInv = [] self.closed = False - def on_close(self, conn): + def on_close(self): self.closed = True - def add_connection(self, conn): - self.conn = conn - - def on_headers(self, conn, message): + def on_headers(self, message): if len(message.headers) > 0: best_header = message.headers[-1] best_header.calc_sha256() self.bestblockhash = best_header.sha256 - def on_getheaders(self, conn, message): + def on_getheaders(self, message): response = self.block_store.headers_for(message.locator, message.hashstop) if response is not None: - conn.send_message(response) + self.send_message(response) - def on_getdata(self, conn, message): - [conn.send_message(r) for r in self.block_store.get_blocks(message.inv)] - [conn.send_message(r) for r in self.tx_store.get_transactions(message.inv)] + def on_getdata(self, message): + [self.send_message(r) for r in self.block_store.get_blocks(message.inv)] + [self.send_message(r) for r in self.tx_store.get_transactions(message.inv)] for i in message.inv: - if i.type == 1: + if i.type == 1 or i.type == 1 | (1 << 30): # MSG_TX or MSG_WITNESS_TX self.tx_request_map[i.hash] = True - elif i.type == 2: + elif i.type == 2 or i.type == 2 | (1 << 30): # MSG_BLOCK or MSG_WITNESS_BLOCK self.block_request_map[i.hash] = True - def on_inv(self, conn, message): + def on_inv(self, message): self.lastInv = [x.hash for x in message.inv] - def on_pong(self, conn, message): + def on_pong(self, message): try: del self.pingMap[message.nonce] except KeyError: raise AssertionError("Got pong for unknown ping [%s]" % repr(message)) - def on_reject(self, conn, message): + def on_reject(self, message): if message.message == b'tx': self.tx_reject_map[message.data] = RejectResult(message.code, message.reason) if message.message == b'block': @@ -102,30 +98,30 @@ class TestNode(NodeConnCB): def send_inv(self, obj): mtype = 2 if isinstance(obj, CBlock) else 1 - self.conn.send_message(msg_inv([CInv(mtype, obj.sha256)])) + self.send_message(msg_inv([CInv(mtype, obj.sha256)])) def send_getheaders(self): # We ask for headers from their last tip. m = msg_getheaders() m.locator = self.block_store.get_locator(self.bestblockhash) - self.conn.send_message(m) + self.send_message(m) def send_header(self, header): m = msg_headers() m.headers.append(header) - self.conn.send_message(m) + self.send_message(m) # This assumes BIP31 def send_ping(self, nonce): self.pingMap[nonce] = True - self.conn.send_message(msg_ping(nonce)) + self.send_message(msg_ping(nonce)) def received_ping_response(self, nonce): return nonce not in self.pingMap def send_mempool(self): self.lastInv = [] - self.conn.send_message(msg_mempool()) + self.send_message(msg_mempool()) # TestInstance: # @@ -156,18 +152,17 @@ class TestNode(NodeConnCB): # across all connections. (If outcome of final tx is specified as true # or false, then only the last tx is tested against outcome.) -class TestInstance(object): +class TestInstance(): def __init__(self, objects=None, sync_every_block=True, sync_every_tx=False): self.blocks_and_transactions = objects if objects else [] self.sync_every_block = sync_every_block self.sync_every_tx = sync_every_tx -class TestManager(object): +class TestManager(): def __init__(self, testgen, datadir): self.test_generator = testgen - self.connections = [] - self.test_nodes = [] + self.p2p_connections= [] self.block_store = BlockStore(datadir) self.tx_store = TxStore(datadir) self.ping_counter = 1 @@ -175,28 +170,24 @@ class TestManager(object): def add_all_connections(self, nodes): for i in range(len(nodes)): # Create a p2p connection to each node - test_node = TestNode(self.block_store, self.tx_store) - self.test_nodes.append(test_node) - self.connections.append(NodeConn('127.0.0.1', p2p_port(i), nodes[i], test_node)) - # Make sure the TestNode (callback class) has a reference to its - # associated NodeConn - test_node.add_connection(self.connections[-1]) + node = TestNode(self.block_store, self.tx_store) + node.peer_connect('127.0.0.1', p2p_port(i)) + self.p2p_connections.append(node) def clear_all_connections(self): - self.connections = [] - self.test_nodes = [] + self.p2p_connections = [] def wait_for_disconnections(self): def disconnected(): - return all(node.closed for node in self.test_nodes) + return all(node.closed for node in self.p2p_connections) wait_until(disconnected, timeout=10, lock=mininode_lock) def wait_for_verack(self): - return all(node.wait_for_verack() for node in self.test_nodes) + return all(node.wait_for_verack() for node in self.p2p_connections) def wait_for_pings(self, counter): def received_pongs(): - return all(node.received_ping_response(counter) for node in self.test_nodes) + return all(node.received_ping_response(counter) for node in self.p2p_connections) wait_until(received_pongs, lock=mininode_lock) # sync_blocks: Wait for all connections to request the blockhash given @@ -206,17 +197,17 @@ class TestManager(object): def blocks_requested(): return all( blockhash in node.block_request_map and node.block_request_map[blockhash] - for node in self.test_nodes + for node in self.p2p_connections ) # --> error if not requested wait_until(blocks_requested, attempts=20*num_blocks, lock=mininode_lock) # Send getheaders message - [ c.cb.send_getheaders() for c in self.connections ] + [ c.send_getheaders() for c in self.p2p_connections ] # Send ping and wait for response -- synchronization hack - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 @@ -226,42 +217,42 @@ class TestManager(object): def transaction_requested(): return all( txhash in node.tx_request_map and node.tx_request_map[txhash] - for node in self.test_nodes + for node in self.p2p_connections ) # --> error if not requested wait_until(transaction_requested, attempts=20*num_events, lock=mininode_lock) # Get the mempool - [ c.cb.send_mempool() for c in self.connections ] + [ c.send_mempool() for c in self.p2p_connections ] # Send ping and wait for response -- synchronization hack - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 # Sort inv responses from each node with mininode_lock: - [ c.cb.lastInv.sort() for c in self.connections ] + [ c.lastInv.sort() for c in self.p2p_connections ] # Verify that the tip of each connection all agree with each other, and # with the expected outcome (if given) def check_results(self, blockhash, outcome): with mininode_lock: - for c in self.connections: + for c in self.p2p_connections: if outcome is None: - if c.cb.bestblockhash != self.connections[0].cb.bestblockhash: + if c.bestblockhash != self.p2p_connections[0].bestblockhash: return False elif isinstance(outcome, RejectResult): # Check that block was rejected w/ code - if c.cb.bestblockhash == blockhash: + if c.bestblockhash == blockhash: return False - if blockhash not in c.cb.block_reject_map: + if blockhash not in c.block_reject_map: logger.error('Block not in reject map: %064x' % (blockhash)) return False - if not outcome.match(c.cb.block_reject_map[blockhash]): - logger.error('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) + if not outcome.match(c.block_reject_map[blockhash]): + logger.error('Block rejected with %s instead of expected %s: %064x' % (c.block_reject_map[blockhash], outcome, blockhash)) return False - elif ((c.cb.bestblockhash == blockhash) != outcome): + elif ((c.bestblockhash == blockhash) != outcome): return False return True @@ -273,21 +264,21 @@ class TestManager(object): # a particular tx's existence in the mempool is the same across all nodes. def check_mempool(self, txhash, outcome): with mininode_lock: - for c in self.connections: + for c in self.p2p_connections: if outcome is None: # Make sure the mempools agree with each other - if c.cb.lastInv != self.connections[0].cb.lastInv: + if c.lastInv != self.p2p_connections[0].lastInv: return False elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code - if txhash in c.cb.lastInv: + if txhash in c.lastInv: return False - if txhash not in c.cb.tx_reject_map: + if txhash not in c.tx_reject_map: logger.error('Tx not in reject map: %064x' % (txhash)) return False - if not outcome.match(c.cb.tx_reject_map[txhash]): - logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) + if not outcome.match(c.tx_reject_map[txhash]): + logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.tx_reject_map[txhash], outcome, txhash)) return False - elif ((txhash in c.cb.lastInv) != outcome): + elif ((txhash in c.lastInv) != outcome): return False return True @@ -295,8 +286,11 @@ class TestManager(object): # Wait until verack is received self.wait_for_verack() - test_number = 1 - for test_instance in self.test_generator.get_tests(): + test_number = 0 + tests = self.test_generator.get_tests() + for test_instance in tests: + test_number += 1 + logger.info("Running test %d: %s line %s" % (test_number, tests.gi_code.co_filename, tests.gi_frame.f_lineno)) # We use these variables to keep track of the last block # and last transaction in the tests, which are used # if we're not syncing on every block or every tx. @@ -329,25 +323,25 @@ class TestManager(object): first_block_with_hash = False with mininode_lock: self.block_store.add_block(block) - for c in self.connections: - if first_block_with_hash and block.sha256 in c.cb.block_request_map and c.cb.block_request_map[block.sha256] == True: + for c in self.p2p_connections: + if first_block_with_hash and block.sha256 in c.block_request_map and c.block_request_map[block.sha256] == True: # There was a previous request for this block hash # Most likely, we delivered a header for this block # but never had the block to respond to the getdata c.send_message(msg_block(block)) else: - c.cb.block_request_map[block.sha256] = False + c.block_request_map[block.sha256] = False # Either send inv's to each node and sync, or add # to invqueue for later inv'ing. if (test_instance.sync_every_block): # if we expect success, send inv and sync every block # if we expect failure, just push the block and see what happens. if outcome == True: - [ c.cb.send_inv(block) for c in self.connections ] + [ c.send_inv(block) for c in self.p2p_connections ] self.sync_blocks(block.sha256, 1) else: - [ c.send_message(msg_block(block)) for c in self.connections ] - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_message(msg_block(block)) for c in self.p2p_connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 if (not self.check_results(tip, outcome)): @@ -357,7 +351,7 @@ class TestManager(object): elif isinstance(b_or_t, CBlockHeader): block_header = b_or_t self.block_store.add_header(block_header) - [ c.cb.send_header(block_header) for c in self.connections ] + [ c.send_header(block_header) for c in self.p2p_connections ] else: # Tx test runner assert(isinstance(b_or_t, CTransaction)) @@ -366,11 +360,11 @@ class TestManager(object): # Add to shared tx store and clear map entry with mininode_lock: self.tx_store.add_transaction(tx) - for c in self.connections: - c.cb.tx_request_map[tx.sha256] = False + for c in self.p2p_connections: + c.tx_request_map[tx.sha256] = False # Again, either inv to all nodes or save for later if (test_instance.sync_every_tx): - [ c.cb.send_inv(tx) for c in self.connections ] + [ c.send_inv(tx) for c in self.p2p_connections ] self.sync_transaction(tx.sha256, 1) if (not self.check_mempool(tx.sha256, outcome)): raise AssertionError("Test failed at test %d" % test_number) @@ -378,29 +372,26 @@ class TestManager(object): invqueue.append(CInv(1, tx.sha256)) # Ensure we're not overflowing the inv queue if len(invqueue) == MAX_INV_SZ: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] # Do final sync if we weren't syncing on every block or every tx. if (not test_instance.sync_every_block and block is not None): if len(invqueue) > 0: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] self.sync_blocks(block.sha256, len(test_instance.blocks_and_transactions)) if (not self.check_results(tip, block_outcome)): raise AssertionError("Block test failed at test %d" % test_number) if (not test_instance.sync_every_tx and tx is not None): if len(invqueue) > 0: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] self.sync_transaction(tx.sha256, len(test_instance.blocks_and_transactions)) if (not self.check_mempool(tx.sha256, tx_outcome)): raise AssertionError("Mempool test failed at test %d" % test_number) - logger.info("Test %d: PASS" % test_number) - test_number += 1 - - [ c.disconnect_node() for c in self.connections ] + [ c.disconnect_node() for c in self.p2p_connections ] self.wait_for_disconnections() self.block_store.close() self.tx_store.close() diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 227b1a17af..ddc3c515b2 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -14,7 +14,7 @@ import os REFERENCE_FILENAME = 'rpc_interface.txt' -class AuthServiceProxyWrapper(object): +class AuthServiceProxyWrapper(): """ An object that wraps AuthServiceProxy to record specific RPC calls. @@ -31,10 +31,11 @@ class AuthServiceProxyWrapper(object): self.auth_service_proxy_instance = auth_service_proxy_instance self.coverage_logfile = coverage_logfile - def __getattr__(self, *args, **kwargs): - return_val = self.auth_service_proxy_instance.__getattr__( - *args, **kwargs) - + def __getattr__(self, name): + return_val = getattr(self.auth_service_proxy_instance, name) + if not isinstance(return_val, type(self.auth_service_proxy_instance)): + # If proxy getattr returned an unwrapped value, do the same here. + return return_val return AuthServiceProxyWrapper(return_val, self.coverage_logfile) def __call__(self, *args, **kwargs): @@ -44,20 +45,23 @@ class AuthServiceProxyWrapper(object): """ return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs) + self._log_call() + return return_val + + def _log_call(self): rpc_method = self.auth_service_proxy_instance._service_name if self.coverage_logfile: with open(self.coverage_logfile, 'a+', encoding='utf8') as f: f.write("%s\n" % rpc_method) - return return_val - - @property - def url(self): - return self.auth_service_proxy_instance.url - def __truediv__(self, relative_uri): - return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri) + return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri, + self.coverage_logfile) + + def get_request(self, *args, **kwargs): + self._log_call() + return self.auth_service_proxy_instance.get_request(*args, **kwargs) def get_filename(dirname, n_node): """ diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 85a6158a2f..aa91fb5b0d 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -84,7 +84,7 @@ def _check_result(val, func, args): ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p ssl.EC_KEY_new_by_curve_name.errcheck = _check_result -class CECKey(object): +class CECKey(): """Wrapper around OpenSSL's EC_KEY""" POINT_CONVERSION_COMPRESSED = 2 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py new file mode 100644 index 0000000000..d8032e4430 --- /dev/null +++ b/test/functional/test_framework/messages.py @@ -0,0 +1,1305 @@ +#!/usr/bin/env python3 +# Copyright (c) 2010 ArtForz -- public domain half-a-node +# Copyright (c) 2012 Jeff Garzik +# Copyright (c) 2010-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Bitcoin test framework primitive and message strcutures + +CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: + data structures that should map to corresponding structures in + bitcoin/primitives + +msg_block, msg_tx, msg_headers, etc.: + data structures that represent network messages + +ser_*, deser_*: functions that handle serialization/deserialization.""" +from codecs import encode +import copy +import hashlib +from io import BytesIO +import random +import socket +import struct +import time + +from test_framework.siphash import siphash256 +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str + +MIN_VERSION_SUPPORTED = 60001 +MY_VERSION = 70014 # past bip-31 for ping/pong +MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) + +MAX_INV_SZ = 50000 +MAX_BLOCK_BASE_SIZE = 1000000 + +COIN = 100000000 # 1 btc in satoshis + +NODE_NETWORK = (1 << 0) +# NODE_GETUTXO = (1 << 1) +NODE_BLOOM = (1 << 2) +NODE_WITNESS = (1 << 3) +NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) +NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) +NODE_NETWORK_LIMITED = (1 << 10) + +# Serialization/deserialization tools +def sha256(s): + return hashlib.new('sha256', s).digest() + +def ripemd160(s): + return hashlib.new('ripemd160', s).digest() + +def hash256(s): + return sha256(sha256(s)) + +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r + +def deser_compact_size(f): + nit = struct.unpack("<B", f.read(1))[0] + if nit == 253: + nit = struct.unpack("<H", f.read(2))[0] + elif nit == 254: + nit = struct.unpack("<I", f.read(4))[0] + elif nit == 255: + nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) + return f.read(nit) + +def ser_string(s): + return ser_compact_size(len(s)) + s + +def deser_uint256(f): + r = 0 + for i in range(8): + t = struct.unpack("<I", f.read(4))[0] + r += t << (i * 32) + return r + + +def ser_uint256(u): + rs = b"" + for i in range(8): + rs += struct.pack("<I", u & 0xFFFFFFFF) + u >>= 32 + return rs + + +def uint256_from_str(s): + r = 0 + t = struct.unpack("<IIIIIIII", s[:32]) + for i in range(8): + r += t[i] << (i * 32) + return r + + +def uint256_from_compact(c): + nbytes = (c >> 24) & 0xFF + v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) + return v + + +def deser_vector(f, c): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = c() + t.deserialize(f) + r.append(t) + return r + + +# ser_function_name: Allow for an alternate serialization function on the +# entries in the vector (we use this for serializing the vector of transactions +# for a witness block). +def ser_vector(l, ser_function_name=None): + r = ser_compact_size(len(l)) + for i in l: + if ser_function_name: + r += getattr(i, ser_function_name)() + else: + r += i.serialize() + return r + + +def deser_uint256_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_uint256(f) + r.append(t) + return r + + +def ser_uint256_vector(l): + r = ser_compact_size(len(l)) + for i in l: + r += ser_uint256(i) + return r + + +def deser_string_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_string(f) + r.append(t) + return r + + +def ser_string_vector(l): + r = ser_compact_size(len(l)) + for sv in l: + r += ser_string(sv) + return r + + +# Deserialize from a hex string representation (eg from RPC) +def FromHex(obj, hex_string): + obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) + return obj + +# Convert a binary-serializable object to hex (eg for submission via RPC) +def ToHex(obj): + return bytes_to_hex_str(obj.serialize()) + +# Objects that map to bitcoind objects, which can be serialized/deserialized + +class CAddress(): + def __init__(self): + self.nServices = 1 + self.pchReserved = b"\x00" * 10 + b"\xff" * 2 + self.ip = "0.0.0.0" + self.port = 0 + + def deserialize(self, f): + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.pchReserved = f.read(12) + self.ip = socket.inet_ntoa(f.read(4)) + self.port = struct.unpack(">H", f.read(2))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nServices) + r += self.pchReserved + r += socket.inet_aton(self.ip) + r += struct.pack(">H", self.port) + return r + + def __repr__(self): + return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, + self.ip, self.port) + +MSG_WITNESS_FLAG = 1<<30 + +class CInv(): + typemap = { + 0: "Error", + 1: "TX", + 2: "Block", + 1|MSG_WITNESS_FLAG: "WitnessTx", + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" + } + + def __init__(self, t=0, h=0): + self.type = t + self.hash = h + + def deserialize(self, f): + self.type = struct.unpack("<i", f.read(4))[0] + self.hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.type) + r += ser_uint256(self.hash) + return r + + def __repr__(self): + return "CInv(type=%s hash=%064x)" \ + % (self.typemap[self.type], self.hash) + + +class CBlockLocator(): + def __init__(self): + self.nVersion = MY_VERSION + self.vHave = [] + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vHave = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256_vector(self.vHave) + return r + + def __repr__(self): + return "CBlockLocator(nVersion=%i vHave=%s)" \ + % (self.nVersion, repr(self.vHave)) + + +class COutPoint(): + def __init__(self, hash=0, n=0): + self.hash = hash + self.n = n + + def deserialize(self, f): + self.hash = deser_uint256(f) + self.n = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += ser_uint256(self.hash) + r += struct.pack("<I", self.n) + return r + + def __repr__(self): + return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) + + +class CTxIn(): + def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): + if outpoint is None: + self.prevout = COutPoint() + else: + self.prevout = outpoint + self.scriptSig = scriptSig + self.nSequence = nSequence + + def deserialize(self, f): + self.prevout = COutPoint() + self.prevout.deserialize(f) + self.scriptSig = deser_string(f) + self.nSequence = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += self.prevout.serialize() + r += ser_string(self.scriptSig) + r += struct.pack("<I", self.nSequence) + return r + + def __repr__(self): + return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ + % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + self.nSequence) + + +class CTxOut(): + def __init__(self, nValue=0, scriptPubKey=b""): + self.nValue = nValue + self.scriptPubKey = scriptPubKey + + def deserialize(self, f): + self.nValue = struct.unpack("<q", f.read(8))[0] + self.scriptPubKey = deser_string(f) + + def serialize(self): + r = b"" + r += struct.pack("<q", self.nValue) + r += ser_string(self.scriptPubKey) + return r + + def __repr__(self): + return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ + % (self.nValue // COIN, self.nValue % COIN, + bytes_to_hex_str(self.scriptPubKey)) + + +class CScriptWitness(): + def __init__(self): + # stack is a vector of strings + self.stack = [] + + def __repr__(self): + return "CScriptWitness(%s)" % \ + (",".join([bytes_to_hex_str(x) for x in self.stack])) + + def is_null(self): + if self.stack: + return False + return True + + +class CTxInWitness(): + def __init__(self): + self.scriptWitness = CScriptWitness() + + def deserialize(self, f): + self.scriptWitness.stack = deser_string_vector(f) + + def serialize(self): + return ser_string_vector(self.scriptWitness.stack) + + def __repr__(self): + return repr(self.scriptWitness) + + def is_null(self): + return self.scriptWitness.is_null() + + +class CTxWitness(): + def __init__(self): + self.vtxinwit = [] + + def deserialize(self, f): + for i in range(len(self.vtxinwit)): + self.vtxinwit[i].deserialize(f) + + def serialize(self): + r = b"" + # This is different than the usual vector serialization -- + # we omit the length of the vector, which is required to be + # the same length as the transaction's vin vector. + for x in self.vtxinwit: + r += x.serialize() + return r + + def __repr__(self): + return "CTxWitness(%s)" % \ + (';'.join([repr(x) for x in self.vtxinwit])) + + def is_null(self): + for x in self.vtxinwit: + if not x.is_null(): + return False + return True + + +class CTransaction(): + def __init__(self, tx=None): + if tx is None: + self.nVersion = 1 + self.vin = [] + self.vout = [] + self.wit = CTxWitness() + self.nLockTime = 0 + self.sha256 = None + self.hash = None + else: + self.nVersion = tx.nVersion + self.vin = copy.deepcopy(tx.vin) + self.vout = copy.deepcopy(tx.vout) + self.nLockTime = tx.nLockTime + self.sha256 = tx.sha256 + self.hash = tx.hash + self.wit = copy.deepcopy(tx.wit) + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vin = deser_vector(f, CTxIn) + flags = 0 + if len(self.vin) == 0: + flags = struct.unpack("<B", f.read(1))[0] + # Not sure why flags can't be zero, but this + # matches the implementation in bitcoind + if (flags != 0): + self.vin = deser_vector(f, CTxIn) + self.vout = deser_vector(f, CTxOut) + else: + self.vout = deser_vector(f, CTxOut) + if flags != 0: + self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] + self.wit.deserialize(f) + self.nLockTime = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize_without_witness(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + r += struct.pack("<I", self.nLockTime) + return r + + # Only serialize with witness when explicitly called for + def serialize_with_witness(self): + flags = 0 + if not self.wit.is_null(): + flags |= 1 + r = b"" + r += struct.pack("<i", self.nVersion) + if flags: + dummy = [] + r += ser_vector(dummy) + r += struct.pack("<B", flags) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + if flags & 1: + if (len(self.wit.vtxinwit) != len(self.vin)): + # vtxinwit must have the same length as vin + self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] + for i in range(len(self.wit.vtxinwit), len(self.vin)): + self.wit.vtxinwit.append(CTxInWitness()) + r += self.wit.serialize() + r += struct.pack("<I", self.nLockTime) + return r + + # Regular serialization is without witness -- must explicitly + # call serialize_with_witness to include witness data. + def serialize(self): + return self.serialize_without_witness() + + # Recalculate the txid (transaction hash without witness) + def rehash(self): + self.sha256 = None + self.calc_sha256() + + # We will only cache the serialization without witness in + # self.sha256 and self.hash -- those are expected to be the txid. + def calc_sha256(self, with_witness=False): + if with_witness: + # Don't cache the result, just return it + return uint256_from_str(hash256(self.serialize_with_witness())) + + if self.sha256 is None: + self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) + self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') + + def is_valid(self): + self.calc_sha256() + for tout in self.vout: + if tout.nValue < 0 or tout.nValue > 21000000 * COIN: + return False + return True + + def __repr__(self): + return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + + +class CBlockHeader(): + def __init__(self, header=None): + if header is None: + self.set_null() + else: + self.nVersion = header.nVersion + self.hashPrevBlock = header.hashPrevBlock + self.hashMerkleRoot = header.hashMerkleRoot + self.nTime = header.nTime + self.nBits = header.nBits + self.nNonce = header.nNonce + self.sha256 = header.sha256 + self.hash = header.hash + self.calc_sha256() + + def set_null(self): + self.nVersion = 1 + self.hashPrevBlock = 0 + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nBits = 0 + self.nNonce = 0 + self.sha256 = None + self.hash = None + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.hashPrevBlock = deser_uint256(f) + self.hashMerkleRoot = deser_uint256(f) + self.nTime = struct.unpack("<I", f.read(4))[0] + self.nBits = struct.unpack("<I", f.read(4))[0] + self.nNonce = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + return r + + def calc_sha256(self): + if self.sha256 is None: + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + self.sha256 = uint256_from_str(hash256(r)) + self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') + + def rehash(self): + self.sha256 = None + self.calc_sha256() + return self.sha256 + + def __repr__(self): + return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce) + + +class CBlock(CBlockHeader): + def __init__(self, header=None): + super(CBlock, self).__init__(header) + self.vtx = [] + + def deserialize(self, f): + super(CBlock, self).deserialize(f) + self.vtx = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += super(CBlock, self).serialize() + if with_witness: + r += ser_vector(self.vtx, "serialize_with_witness") + else: + r += ser_vector(self.vtx) + return r + + # Calculate the merkle root given a vector of transaction hashes + @classmethod + def get_merkle_root(cls, hashes): + while len(hashes) > 1: + newhashes = [] + for i in range(0, len(hashes), 2): + i2 = min(i+1, len(hashes)-1) + newhashes.append(hash256(hashes[i] + hashes[i2])) + hashes = newhashes + return uint256_from_str(hashes[0]) + + def calc_merkle_root(self): + hashes = [] + for tx in self.vtx: + tx.calc_sha256() + hashes.append(ser_uint256(tx.sha256)) + return self.get_merkle_root(hashes) + + def calc_witness_merkle_root(self): + # For witness root purposes, the hash of the + # coinbase, with witness, is defined to be 0...0 + hashes = [ser_uint256(0)] + + for tx in self.vtx[1:]: + # Calculate the hashes with witness data + hashes.append(ser_uint256(tx.calc_sha256(True))) + + return self.get_merkle_root(hashes) + + def is_valid(self): + self.calc_sha256() + target = uint256_from_compact(self.nBits) + if self.sha256 > target: + return False + for tx in self.vtx: + if not tx.is_valid(): + return False + if self.calc_merkle_root() != self.hashMerkleRoot: + return False + return True + + def solve(self): + self.rehash() + target = uint256_from_compact(self.nBits) + while self.sha256 > target: + self.nNonce += 1 + self.rehash() + + def __repr__(self): + return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) + + +class PrefilledTransaction(): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def serialize_with_witness(self): + return self.serialize(with_witness=True) + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + # When using version 2 compact blocks, we must serialize with_witness. + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + if with_witness: + r += ser_vector(self.prefilled_txn, "serialize_with_witness") + else: + r += ser_vector(self.prefilled_txn) + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + +# P2P version of the above that will use witness serialization (for compact +# block version 2) +class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): + def serialize(self): + return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + self.use_witness = False + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + if self.use_witness: + ret = P2PHeaderAndShortWitnessIDs() + else: + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + # Version 2 compact blocks use wtxid in shortids (rather than txid) + def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + self.use_witness = use_witness + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + tx_hash = block.vtx[i].sha256 + if use_witness: + tx_hash = block.vtx[i].calc_sha256(with_witness=True) + self.shortids.append(calculate_shortid(k0, k1, tx_hash)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions) + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + +# Objects that correspond to messages on the wire +class msg_version(): + command = b"version" + + def __init__(self): + self.nVersion = MY_VERSION + self.nServices = NODE_NETWORK | NODE_WITNESS + self.nTime = int(time.time()) + self.addrTo = CAddress() + self.addrFrom = CAddress() + self.nNonce = random.getrandbits(64) + self.strSubVer = MY_SUBVERSION + self.nStartingHeight = -1 + self.nRelay = MY_RELAY + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + if self.nVersion == 10300: + self.nVersion = 300 + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.nTime = struct.unpack("<q", f.read(8))[0] + self.addrTo = CAddress() + self.addrTo.deserialize(f) + + if self.nVersion >= 106: + self.addrFrom = CAddress() + self.addrFrom.deserialize(f) + self.nNonce = struct.unpack("<Q", f.read(8))[0] + self.strSubVer = deser_string(f) + else: + self.addrFrom = None + self.nNonce = None + self.strSubVer = None + self.nStartingHeight = None + + if self.nVersion >= 209: + self.nStartingHeight = struct.unpack("<i", f.read(4))[0] + else: + self.nStartingHeight = None + + if self.nVersion >= 70001: + # Relay field is optional for version 70001 onwards + try: + self.nRelay = struct.unpack("<b", f.read(1))[0] + except: + self.nRelay = 0 + else: + self.nRelay = 0 + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += struct.pack("<Q", self.nServices) + r += struct.pack("<q", self.nTime) + r += self.addrTo.serialize() + r += self.addrFrom.serialize() + r += struct.pack("<Q", self.nNonce) + r += ser_string(self.strSubVer) + r += struct.pack("<i", self.nStartingHeight) + r += struct.pack("<b", self.nRelay) + return r + + def __repr__(self): + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ + % (self.nVersion, self.nServices, time.ctime(self.nTime), + repr(self.addrTo), repr(self.addrFrom), self.nNonce, + self.strSubVer, self.nStartingHeight, self.nRelay) + + +class msg_verack(): + command = b"verack" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_verack()" + + +class msg_addr(): + command = b"addr" + + def __init__(self): + self.addrs = [] + + def deserialize(self, f): + self.addrs = deser_vector(f, CAddress) + + def serialize(self): + return ser_vector(self.addrs) + + def __repr__(self): + return "msg_addr(addrs=%s)" % (repr(self.addrs)) + + +class msg_inv(): + command = b"inv" + + def __init__(self, inv=None): + if inv is None: + self.inv = [] + else: + self.inv = inv + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_inv(inv=%s)" % (repr(self.inv)) + + +class msg_getdata(): + command = b"getdata" + + def __init__(self, inv=None): + self.inv = inv if inv != None else [] + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_getdata(inv=%s)" % (repr(self.inv)) + + +class msg_getblocks(): + command = b"getblocks" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getblocks(locator=%s hashstop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +class msg_tx(): + command = b"tx" + + def __init__(self, tx=CTransaction()): + self.tx = tx + + def deserialize(self, f): + self.tx.deserialize(f) + + def serialize(self): + return self.tx.serialize_without_witness() + + def __repr__(self): + return "msg_tx(tx=%s)" % (repr(self.tx)) + +class msg_witness_tx(msg_tx): + + def serialize(self): + return self.tx.serialize_with_witness() + + +class msg_block(): + command = b"block" + + def __init__(self, block=None): + if block is None: + self.block = CBlock() + else: + self.block = block + + def deserialize(self, f): + self.block.deserialize(f) + + def serialize(self): + return self.block.serialize() + + def __repr__(self): + return "msg_block(block=%s)" % (repr(self.block)) + +# for cases where a user needs tighter control over what is sent over the wire +# note that the user must supply the name of the command, and the data +class msg_generic(): + def __init__(self, command, data=None): + self.command = command + self.data = data + + def serialize(self): + return self.data + + def __repr__(self): + return "msg_generic()" + +class msg_witness_block(msg_block): + + def serialize(self): + r = self.block.serialize(with_witness=True) + return r + +class msg_getaddr(): + command = b"getaddr" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_getaddr()" + + +class msg_ping(): + command = b"ping" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_ping(nonce=%08x)" % self.nonce + + +class msg_pong(): + command = b"pong" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_pong(nonce=%08x)" % self.nonce + + +class msg_mempool(): + command = b"mempool" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_mempool()" + +class msg_sendheaders(): + command = b"sendheaders" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_sendheaders()" + + +# getheaders message has +# number of entries +# vector of hashes +# hash_stop (hash of last desired block header, 0 to get as many as possible) +class msg_getheaders(): + command = b"getheaders" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getheaders(locator=%s, stop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +# headers message has +# <count> <vector of block headers> +class msg_headers(): + command = b"headers" + + def __init__(self, headers=None): + self.headers = headers if headers is not None else [] + + def deserialize(self, f): + # comment in bitcoind indicates these should be deserialized as blocks + blocks = deser_vector(f, CBlock) + for x in blocks: + self.headers.append(CBlockHeader(x)) + + def serialize(self): + blocks = [CBlock(x) for x in self.headers] + return ser_vector(blocks) + + def __repr__(self): + return "msg_headers(headers=%s)" % repr(self.headers) + + +class msg_reject(): + command = b"reject" + REJECT_MALFORMED = 1 + + def __init__(self): + self.message = b"" + self.code = 0 + self.reason = b"" + self.data = 0 + + def deserialize(self, f): + self.message = deser_string(f) + self.code = struct.unpack("<B", f.read(1))[0] + self.reason = deser_string(f) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + self.data = deser_uint256(f) + + def serialize(self): + r = ser_string(self.message) + r += struct.pack("<B", self.code) + r += ser_string(self.reason) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + r += ser_uint256(self.data) + return r + + def __repr__(self): + return "msg_reject: %s %d %s [%064x]" \ + % (self.message, self.code, self.reason, self.data) + +class msg_feefilter(): + command = b"feefilter" + + def __init__(self, feerate=0): + self.feerate = feerate + + def deserialize(self, f): + self.feerate = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.feerate) + return r + + def __repr__(self): + return "msg_feefilter(feerate=%08x)" % self.feerate + +class msg_sendcmpct(): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize() + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + +class msg_witness_blocktxn(msg_blocktxn): + def serialize(self): + r = b"" + r += self.block_transactions.serialize(with_witness=True) + return r diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index c6f596156a..724d418099 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -9,1462 +9,252 @@ This python code was modified from ArtForz' public domain half-a-node, as found in the mini-node branch of http://github.com/jgarzik/pynode. -NodeConn: an object which manages p2p connectivity to a bitcoin node -NodeConnCB: a base class that describes the interface for receiving - callbacks with network messages from a NodeConn -CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: - data structures that should map to corresponding structures in - bitcoin/primitives -msg_block, msg_tx, msg_headers, etc.: - data structures that represent network messages -ser_*, deser_*: functions that handle serialization/deserialization -""" - +P2PConnection: A low-level connection object to a node's P2P interface +P2PInterface: A high-level interface object for communicating to a node over P2P""" import asyncore -from codecs import encode from collections import defaultdict -import copy -import hashlib from io import BytesIO import logging -import random import socket import struct import sys -import time -from threading import RLock, Thread - -from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until - -BIP0031_VERSION = 60000 -MY_VERSION = 70014 # past bip-31 for ping/pong -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" -MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) - -MAX_INV_SZ = 50000 -MAX_BLOCK_BASE_SIZE = 1000000 +import threading -COIN = 100000000 # 1 btc in satoshis - -NODE_NETWORK = (1 << 0) -# NODE_GETUTXO = (1 << 1) -# NODE_BLOOM = (1 << 2) -NODE_WITNESS = (1 << 3) -NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) -NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) +from test_framework.messages import * +from test_framework.util import wait_until logger = logging.getLogger("TestFramework.mininode") -# Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when -# using select) -mininode_socket_map = dict() +MESSAGEMAP = { + b"addr": msg_addr, + b"block": msg_block, + b"blocktxn": msg_blocktxn, + b"cmpctblock": msg_cmpctblock, + b"feefilter": msg_feefilter, + b"getaddr": msg_getaddr, + b"getblocks": msg_getblocks, + b"getblocktxn": msg_getblocktxn, + b"getdata": msg_getdata, + b"getheaders": msg_getheaders, + b"headers": msg_headers, + b"inv": msg_inv, + b"mempool": msg_mempool, + b"ping": msg_ping, + b"pong": msg_pong, + b"reject": msg_reject, + b"sendcmpct": msg_sendcmpct, + b"sendheaders": msg_sendheaders, + b"tx": msg_tx, + b"verack": msg_verack, + b"version": msg_version, +} + +MAGIC_BYTES = { + "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet + "testnet3": b"\x0b\x11\x09\x07", # testnet3 + "regtest": b"\xfa\xbf\xb5\xda", # regtest +} + +class P2PConnection(asyncore.dispatcher): + """A low-level connection object to a node's P2P interface. + + This class is responsible for: + + - opening and closing the TCP connection to the node + - reading bytes from and writing bytes to the socket + - deserializing and serializing the P2P message header + - logging messages as they are sent and received + + This class contains no logic for handing the P2P message payloads. It must be + sub-classed and the on_message() callback overridden.""" -# One lock for synchronizing all data access between the networking thread (see -# NetworkThread below) and the thread running the test logic. For simplicity, -# NodeConn acquires this lock whenever delivering a message to a NodeConnCB, -# and whenever adding anything to the send buffer (in send_message()). This -# lock should be acquired in the thread running the test logic to synchronize -# access to any data shared with the NodeConnCB or NodeConn. -mininode_lock = RLock() - -# Serialization/deserialization tools -def sha256(s): - return hashlib.new('sha256', s).digest() - -def ripemd160(s): - return hashlib.new('ripemd160', s).digest() - -def hash256(s): - return sha256(sha256(s)) - -def ser_compact_size(l): - r = b"" - if l < 253: - r = struct.pack("B", l) - elif l < 0x10000: - r = struct.pack("<BH", 253, l) - elif l < 0x100000000: - r = struct.pack("<BI", 254, l) - else: - r = struct.pack("<BQ", 255, l) - return r - -def deser_compact_size(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] - return nit - -def deser_string(f): - nit = deser_compact_size(f) - return f.read(nit) - -def ser_string(s): - return ser_compact_size(len(s)) + s - -def deser_uint256(f): - r = 0 - for i in range(8): - t = struct.unpack("<I", f.read(4))[0] - r += t << (i * 32) - return r - - -def ser_uint256(u): - rs = b"" - for i in range(8): - rs += struct.pack("<I", u & 0xFFFFFFFF) - u >>= 32 - return rs - - -def uint256_from_str(s): - r = 0 - t = struct.unpack("<IIIIIIII", s[:32]) - for i in range(8): - r += t[i] << (i * 32) - return r - - -def uint256_from_compact(c): - nbytes = (c >> 24) & 0xFF - v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) - return v - - -def deser_vector(f, c): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = c() - t.deserialize(f) - r.append(t) - return r - - -# ser_function_name: Allow for an alternate serialization function on the -# entries in the vector (we use this for serializing the vector of transactions -# for a witness block). -def ser_vector(l, ser_function_name=None): - r = ser_compact_size(len(l)) - for i in l: - if ser_function_name: - r += getattr(i, ser_function_name)() - else: - r += i.serialize() - return r - - -def deser_uint256_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_uint256(f) - r.append(t) - return r - - -def ser_uint256_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += ser_uint256(i) - return r - - -def deser_string_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_string(f) - r.append(t) - return r - - -def ser_string_vector(l): - r = ser_compact_size(len(l)) - for sv in l: - r += ser_string(sv) - return r - - -def deser_int_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = struct.unpack("<i", f.read(4))[0] - r.append(t) - return r - - -def ser_int_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += struct.pack("<i", i) - return r - -# Deserialize from a hex string representation (eg from RPC) -def FromHex(obj, hex_string): - obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) - return obj - -# Convert a binary-serializable object to hex (eg for submission via RPC) -def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) - -# Objects that map to bitcoind objects, which can be serialized/deserialized - -class CAddress(object): def __init__(self): - self.nServices = 1 - self.pchReserved = b"\x00" * 10 + b"\xff" * 2 - self.ip = "0.0.0.0" - self.port = 0 - - def deserialize(self, f): - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.pchReserved = f.read(12) - self.ip = socket.inet_ntoa(f.read(4)) - self.port = struct.unpack(">H", f.read(2))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nServices) - r += self.pchReserved - r += socket.inet_aton(self.ip) - r += struct.pack(">H", self.port) - return r - - def __repr__(self): - return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, - self.ip, self.port) - -MSG_WITNESS_FLAG = 1<<30 - -class CInv(object): - typemap = { - 0: "Error", - 1: "TX", - 2: "Block", - 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock", - 4: "CompactBlock" - } - - def __init__(self, t=0, h=0): - self.type = t - self.hash = h - - def deserialize(self, f): - self.type = struct.unpack("<i", f.read(4))[0] - self.hash = deser_uint256(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.type) - r += ser_uint256(self.hash) - return r - - def __repr__(self): - return "CInv(type=%s hash=%064x)" \ - % (self.typemap[self.type], self.hash) - - -class CBlockLocator(object): - def __init__(self): - self.nVersion = MY_VERSION - self.vHave = [] - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vHave = deser_uint256_vector(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256_vector(self.vHave) - return r - - def __repr__(self): - return "CBlockLocator(nVersion=%i vHave=%s)" \ - % (self.nVersion, repr(self.vHave)) - - -class COutPoint(object): - def __init__(self, hash=0, n=0): - self.hash = hash - self.n = n - - def deserialize(self, f): - self.hash = deser_uint256(f) - self.n = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += ser_uint256(self.hash) - r += struct.pack("<I", self.n) - return r - - def __repr__(self): - return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) - - -class CTxIn(object): - def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): - if outpoint is None: - self.prevout = COutPoint() - else: - self.prevout = outpoint - self.scriptSig = scriptSig - self.nSequence = nSequence - - def deserialize(self, f): - self.prevout = COutPoint() - self.prevout.deserialize(f) - self.scriptSig = deser_string(f) - self.nSequence = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += self.prevout.serialize() - r += ser_string(self.scriptSig) - r += struct.pack("<I", self.nSequence) - return r - - def __repr__(self): - return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), - self.nSequence) - - -class CTxOut(object): - def __init__(self, nValue=0, scriptPubKey=b""): - self.nValue = nValue - self.scriptPubKey = scriptPubKey - - def deserialize(self, f): - self.nValue = struct.unpack("<q", f.read(8))[0] - self.scriptPubKey = deser_string(f) - - def serialize(self): - r = b"" - r += struct.pack("<q", self.nValue) - r += ser_string(self.scriptPubKey) - return r - - def __repr__(self): - return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ - % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) - - -class CScriptWitness(object): - def __init__(self): - # stack is a vector of strings - self.stack = [] - - def __repr__(self): - return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) - - def is_null(self): - if self.stack: - return False - return True - - -class CTxInWitness(object): - def __init__(self): - self.scriptWitness = CScriptWitness() - - def deserialize(self, f): - self.scriptWitness.stack = deser_string_vector(f) - - def serialize(self): - return ser_string_vector(self.scriptWitness.stack) - - def __repr__(self): - return repr(self.scriptWitness) - - def is_null(self): - return self.scriptWitness.is_null() - - -class CTxWitness(object): - def __init__(self): - self.vtxinwit = [] - - def deserialize(self, f): - for i in range(len(self.vtxinwit)): - self.vtxinwit[i].deserialize(f) - - def serialize(self): - r = b"" - # This is different than the usual vector serialization -- - # we omit the length of the vector, which is required to be - # the same length as the transaction's vin vector. - for x in self.vtxinwit: - r += x.serialize() - return r - - def __repr__(self): - return "CTxWitness(%s)" % \ - (';'.join([repr(x) for x in self.vtxinwit])) - - def is_null(self): - for x in self.vtxinwit: - if not x.is_null(): - return False - return True - - -class CTransaction(object): - def __init__(self, tx=None): - if tx is None: - self.nVersion = 1 - self.vin = [] - self.vout = [] - self.wit = CTxWitness() - self.nLockTime = 0 - self.sha256 = None - self.hash = None - else: - self.nVersion = tx.nVersion - self.vin = copy.deepcopy(tx.vin) - self.vout = copy.deepcopy(tx.vout) - self.nLockTime = tx.nLockTime - self.sha256 = tx.sha256 - self.hash = tx.hash - self.wit = copy.deepcopy(tx.wit) - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vin = deser_vector(f, CTxIn) - flags = 0 - if len(self.vin) == 0: - flags = struct.unpack("<B", f.read(1))[0] - # Not sure why flags can't be zero, but this - # matches the implementation in bitcoind - if (flags != 0): - self.vin = deser_vector(f, CTxIn) - self.vout = deser_vector(f, CTxOut) - else: - self.vout = deser_vector(f, CTxOut) - if flags != 0: - self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] - self.wit.deserialize(f) - self.nLockTime = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize_without_witness(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - r += struct.pack("<I", self.nLockTime) - return r - - # Only serialize with witness when explicitly called for - def serialize_with_witness(self): - flags = 0 - if not self.wit.is_null(): - flags |= 1 - r = b"" - r += struct.pack("<i", self.nVersion) - if flags: - dummy = [] - r += ser_vector(dummy) - r += struct.pack("<B", flags) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - if flags & 1: - if (len(self.wit.vtxinwit) != len(self.vin)): - # vtxinwit must have the same length as vin - self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] - for i in range(len(self.wit.vtxinwit), len(self.vin)): - self.wit.vtxinwit.append(CTxInWitness()) - r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) - return r - - # Regular serialization is without witness -- must explicitly - # call serialize_with_witness to include witness data. - def serialize(self): - return self.serialize_without_witness() - - # Recalculate the txid (transaction hash without witness) - def rehash(self): - self.sha256 = None - self.calc_sha256() - - # We will only cache the serialization without witness in - # self.sha256 and self.hash -- those are expected to be the txid. - def calc_sha256(self, with_witness=False): - if with_witness: - # Don't cache the result, just return it - return uint256_from_str(hash256(self.serialize_with_witness())) - - if self.sha256 is None: - self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) - self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') - - def is_valid(self): - self.calc_sha256() - for tout in self.vout: - if tout.nValue < 0 or tout.nValue > 21000000 * COIN: - return False - return True - - def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) - - -class CBlockHeader(object): - def __init__(self, header=None): - if header is None: - self.set_null() - else: - self.nVersion = header.nVersion - self.hashPrevBlock = header.hashPrevBlock - self.hashMerkleRoot = header.hashMerkleRoot - self.nTime = header.nTime - self.nBits = header.nBits - self.nNonce = header.nNonce - self.sha256 = header.sha256 - self.hash = header.hash - self.calc_sha256() - - def set_null(self): - self.nVersion = 1 - self.hashPrevBlock = 0 - self.hashMerkleRoot = 0 - self.nTime = 0 - self.nBits = 0 - self.nNonce = 0 - self.sha256 = None - self.hash = None - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.hashPrevBlock = deser_uint256(f) - self.hashMerkleRoot = deser_uint256(f) - self.nTime = struct.unpack("<I", f.read(4))[0] - self.nBits = struct.unpack("<I", f.read(4))[0] - self.nNonce = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - return r - - def calc_sha256(self): - if self.sha256 is None: - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - self.sha256 = uint256_from_str(hash256(r)) - self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') - - def rehash(self): - self.sha256 = None - self.calc_sha256() - return self.sha256 - - def __repr__(self): - return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce) - - -class CBlock(CBlockHeader): - def __init__(self, header=None): - super(CBlock, self).__init__(header) - self.vtx = [] - - def deserialize(self, f): - super(CBlock, self).deserialize(f) - self.vtx = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += super(CBlock, self).serialize() - if with_witness: - r += ser_vector(self.vtx, "serialize_with_witness") - else: - r += ser_vector(self.vtx) - return r - - # Calculate the merkle root given a vector of transaction hashes - @classmethod - def get_merkle_root(cls, hashes): - while len(hashes) > 1: - newhashes = [] - for i in range(0, len(hashes), 2): - i2 = min(i+1, len(hashes)-1) - newhashes.append(hash256(hashes[i] + hashes[i2])) - hashes = newhashes - return uint256_from_str(hashes[0]) - - def calc_merkle_root(self): - hashes = [] - for tx in self.vtx: - tx.calc_sha256() - hashes.append(ser_uint256(tx.sha256)) - return self.get_merkle_root(hashes) - - def calc_witness_merkle_root(self): - # For witness root purposes, the hash of the - # coinbase, with witness, is defined to be 0...0 - hashes = [ser_uint256(0)] - - for tx in self.vtx[1:]: - # Calculate the hashes with witness data - hashes.append(ser_uint256(tx.calc_sha256(True))) - - return self.get_merkle_root(hashes) - - def is_valid(self): - self.calc_sha256() - target = uint256_from_compact(self.nBits) - if self.sha256 > target: - return False - for tx in self.vtx: - if not tx.is_valid(): - return False - if self.calc_merkle_root() != self.hashMerkleRoot: - return False - return True - - def solve(self): - self.rehash() - target = uint256_from_compact(self.nBits) - while self.sha256 > target: - self.nNonce += 1 - self.rehash() - - def __repr__(self): - return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) - - -class CUnsignedAlert(object): - def __init__(self): - self.nVersion = 1 - self.nRelayUntil = 0 - self.nExpiration = 0 - self.nID = 0 - self.nCancel = 0 - self.setCancel = [] - self.nMinVer = 0 - self.nMaxVer = 0 - self.setSubVer = [] - self.nPriority = 0 - self.strComment = b"" - self.strStatusBar = b"" - self.strReserved = b"" - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.nRelayUntil = struct.unpack("<q", f.read(8))[0] - self.nExpiration = struct.unpack("<q", f.read(8))[0] - self.nID = struct.unpack("<i", f.read(4))[0] - self.nCancel = struct.unpack("<i", f.read(4))[0] - self.setCancel = deser_int_vector(f) - self.nMinVer = struct.unpack("<i", f.read(4))[0] - self.nMaxVer = struct.unpack("<i", f.read(4))[0] - self.setSubVer = deser_string_vector(f) - self.nPriority = struct.unpack("<i", f.read(4))[0] - self.strComment = deser_string(f) - self.strStatusBar = deser_string(f) - self.strReserved = deser_string(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += struct.pack("<q", self.nRelayUntil) - r += struct.pack("<q", self.nExpiration) - r += struct.pack("<i", self.nID) - r += struct.pack("<i", self.nCancel) - r += ser_int_vector(self.setCancel) - r += struct.pack("<i", self.nMinVer) - r += struct.pack("<i", self.nMaxVer) - r += ser_string_vector(self.setSubVer) - r += struct.pack("<i", self.nPriority) - r += ser_string(self.strComment) - r += ser_string(self.strStatusBar) - r += ser_string(self.strReserved) - return r - - def __repr__(self): - return "CUnsignedAlert(nVersion %d, nRelayUntil %d, nExpiration %d, nID %d, nCancel %d, nMinVer %d, nMaxVer %d, nPriority %d, strComment %s, strStatusBar %s, strReserved %s)" \ - % (self.nVersion, self.nRelayUntil, self.nExpiration, self.nID, - self.nCancel, self.nMinVer, self.nMaxVer, self.nPriority, - self.strComment, self.strStatusBar, self.strReserved) - - -class CAlert(object): - def __init__(self): - self.vchMsg = b"" - self.vchSig = b"" - - def deserialize(self, f): - self.vchMsg = deser_string(f) - self.vchSig = deser_string(f) - - def serialize(self): - r = b"" - r += ser_string(self.vchMsg) - r += ser_string(self.vchSig) - return r - - def __repr__(self): - return "CAlert(vchMsg.sz %d, vchSig.sz %d)" \ - % (len(self.vchMsg), len(self.vchSig)) - - -class PrefilledTransaction(object): - def __init__(self, index=0, tx = None): - self.index = index - self.tx = tx - - def deserialize(self, f): - self.index = deser_compact_size(f) - self.tx = CTransaction() - self.tx.deserialize(f) - - def serialize(self, with_witness=False): - r = b"" - r += ser_compact_size(self.index) - if with_witness: - r += self.tx.serialize_with_witness() - else: - r += self.tx.serialize_without_witness() - return r - - def serialize_with_witness(self): - return self.serialize(with_witness=True) - - def __repr__(self): - return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) - -# This is what we send on the wire, in a cmpctblock message. -class P2PHeaderAndShortIDs(object): - def __init__(self): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids_length = 0 - self.shortids = [] - self.prefilled_txn_length = 0 - self.prefilled_txn = [] - - def deserialize(self, f): - self.header.deserialize(f) - self.nonce = struct.unpack("<Q", f.read(8))[0] - self.shortids_length = deser_compact_size(f) - for i in range(self.shortids_length): - # shortids are defined to be 6 bytes in the spec, so append - # two zero bytes and read it in as an 8-byte number - self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) - self.prefilled_txn = deser_vector(f, PrefilledTransaction) - self.prefilled_txn_length = len(self.prefilled_txn) - - # When using version 2 compact blocks, we must serialize with_witness. - def serialize(self, with_witness=False): - r = b"" - r += self.header.serialize() - r += struct.pack("<Q", self.nonce) - r += ser_compact_size(self.shortids_length) - for x in self.shortids: - # We only want the first 6 bytes - r += struct.pack("<Q", x)[0:6] - if with_witness: - r += ser_vector(self.prefilled_txn, "serialize_with_witness") - else: - r += ser_vector(self.prefilled_txn) - return r - - def __repr__(self): - return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) - -# P2P version of the above that will use witness serialization (for compact -# block version 2) -class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): - def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) - -# Calculate the BIP 152-compact blocks shortid for a given transaction hash -def calculate_shortid(k0, k1, tx_hash): - expected_shortid = siphash256(k0, k1, tx_hash) - expected_shortid &= 0x0000ffffffffffff - return expected_shortid - -# This version gets rid of the array lengths, and reinterprets the differential -# encoding into indices that can be used for lookup. -class HeaderAndShortIDs(object): - def __init__(self, p2pheaders_and_shortids = None): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids = [] - self.prefilled_txn = [] - self.use_witness = False - - if p2pheaders_and_shortids != None: - self.header = p2pheaders_and_shortids.header - self.nonce = p2pheaders_and_shortids.nonce - self.shortids = p2pheaders_and_shortids.shortids - last_index = -1 - for x in p2pheaders_and_shortids.prefilled_txn: - self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) - last_index = self.prefilled_txn[-1].index - - def to_p2p(self): - if self.use_witness: - ret = P2PHeaderAndShortWitnessIDs() - else: - ret = P2PHeaderAndShortIDs() - ret.header = self.header - ret.nonce = self.nonce - ret.shortids_length = len(self.shortids) - ret.shortids = self.shortids - ret.prefilled_txn_length = len(self.prefilled_txn) - ret.prefilled_txn = [] - last_index = -1 - for x in self.prefilled_txn: - ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) - last_index = x.index - return ret - - def get_siphash_keys(self): - header_nonce = self.header.serialize() - header_nonce += struct.pack("<Q", self.nonce) - hash_header_nonce_as_str = sha256(header_nonce) - key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] - key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] - return [ key0, key1 ] - - # Version 2 compact blocks use wtxid in shortids (rather than txid) - def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): - self.header = CBlockHeader(block) - self.nonce = nonce - self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] - self.shortids = [] - self.use_witness = use_witness - [k0, k1] = self.get_siphash_keys() - for i in range(len(block.vtx)): - if i not in prefill_list: - tx_hash = block.vtx[i].sha256 - if use_witness: - tx_hash = block.vtx[i].calc_sha256(with_witness=True) - self.shortids.append(calculate_shortid(k0, k1, tx_hash)) - - def __repr__(self): - return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) - - -class BlockTransactionsRequest(object): - - def __init__(self, blockhash=0, indexes = None): - self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - indexes_length = deser_compact_size(f) - for i in range(indexes_length): - self.indexes.append(deser_compact_size(f)) - - def serialize(self): - r = b"" - r += ser_uint256(self.blockhash) - r += ser_compact_size(len(self.indexes)) - for x in self.indexes: - r += ser_compact_size(x) - return r - - # helper to set the differentially encoded indexes from absolute ones - def from_absolute(self, absolute_indexes): - self.indexes = [] - last_index = -1 - for x in absolute_indexes: - self.indexes.append(x-last_index-1) - last_index = x - - def to_absolute(self): - absolute_indexes = [] - last_index = -1 - for x in self.indexes: - absolute_indexes.append(x+last_index+1) - last_index = absolute_indexes[-1] - return absolute_indexes - - def __repr__(self): - return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) - - -class BlockTransactions(object): - - def __init__(self, blockhash=0, transactions = None): - self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - self.transactions = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += ser_uint256(self.blockhash) - if with_witness: - r += ser_vector(self.transactions, "serialize_with_witness") - else: - r += ser_vector(self.transactions) - return r - - def __repr__(self): - return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) - - -# Objects that correspond to messages on the wire -class msg_version(object): - command = b"version" - - def __init__(self): - self.nVersion = MY_VERSION - self.nServices = 1 - self.nTime = int(time.time()) - self.addrTo = CAddress() - self.addrFrom = CAddress() - self.nNonce = random.getrandbits(64) - self.strSubVer = MY_SUBVERSION - self.nStartingHeight = -1 - self.nRelay = MY_RELAY - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - if self.nVersion == 10300: - self.nVersion = 300 - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.nTime = struct.unpack("<q", f.read(8))[0] - self.addrTo = CAddress() - self.addrTo.deserialize(f) - - if self.nVersion >= 106: - self.addrFrom = CAddress() - self.addrFrom.deserialize(f) - self.nNonce = struct.unpack("<Q", f.read(8))[0] - self.strSubVer = deser_string(f) - else: - self.addrFrom = None - self.nNonce = None - self.strSubVer = None - self.nStartingHeight = None - - if self.nVersion >= 209: - self.nStartingHeight = struct.unpack("<i", f.read(4))[0] - else: - self.nStartingHeight = None - - if self.nVersion >= 70001: - # Relay field is optional for version 70001 onwards - try: - self.nRelay = struct.unpack("<b", f.read(1))[0] - except: - self.nRelay = 0 - else: - self.nRelay = 0 - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += struct.pack("<Q", self.nServices) - r += struct.pack("<q", self.nTime) - r += self.addrTo.serialize() - r += self.addrFrom.serialize() - r += struct.pack("<Q", self.nNonce) - r += ser_string(self.strSubVer) - r += struct.pack("<i", self.nStartingHeight) - r += struct.pack("<b", self.nRelay) - return r - - def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ - % (self.nVersion, self.nServices, time.ctime(self.nTime), - repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight, self.nRelay) - - -class msg_verack(object): - command = b"verack" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_verack()" - - -class msg_addr(object): - command = b"addr" - - def __init__(self): - self.addrs = [] - - def deserialize(self, f): - self.addrs = deser_vector(f, CAddress) - - def serialize(self): - return ser_vector(self.addrs) - - def __repr__(self): - return "msg_addr(addrs=%s)" % (repr(self.addrs)) - + # All P2PConnections must be created before starting the NetworkThread. + # assert that the network thread is not running. + assert not network_thread_running() -class msg_alert(object): - command = b"alert" + super().__init__(map=mininode_socket_map) - def __init__(self): - self.alert = CAlert() - - def deserialize(self, f): - self.alert = CAlert() - self.alert.deserialize(f) - - def serialize(self): - r = b"" - r += self.alert.serialize() - return r - - def __repr__(self): - return "msg_alert(alert=%s)" % (repr(self.alert), ) - - -class msg_inv(object): - command = b"inv" - - def __init__(self, inv=None): - if inv is None: - self.inv = [] - else: - self.inv = inv - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_inv(inv=%s)" % (repr(self.inv)) - - -class msg_getdata(object): - command = b"getdata" - - def __init__(self, inv=None): - self.inv = inv if inv != None else [] - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_getdata(inv=%s)" % (repr(self.inv)) - - -class msg_getblocks(object): - command = b"getblocks" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getblocks(locator=%s hashstop=%064x)" \ - % (repr(self.locator), self.hashstop) - - -class msg_tx(object): - command = b"tx" - - def __init__(self, tx=CTransaction()): - self.tx = tx - - def deserialize(self, f): - self.tx.deserialize(f) - - def serialize(self): - return self.tx.serialize_without_witness() - - def __repr__(self): - return "msg_tx(tx=%s)" % (repr(self.tx)) - -class msg_witness_tx(msg_tx): - - def serialize(self): - return self.tx.serialize_with_witness() - - -class msg_block(object): - command = b"block" - - def __init__(self, block=None): - if block is None: - self.block = CBlock() - else: - self.block = block - - def deserialize(self, f): - self.block.deserialize(f) - - def serialize(self): - return self.block.serialize() - - def __repr__(self): - return "msg_block(block=%s)" % (repr(self.block)) - -# for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data -class msg_generic(object): - def __init__(self, command, data=None): - self.command = command - self.data = data - - def serialize(self): - return self.data - - def __repr__(self): - return "msg_generic()" - -class msg_witness_block(msg_block): - - def serialize(self): - r = self.block.serialize(with_witness=True) - return r - -class msg_getaddr(object): - command = b"getaddr" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_getaddr()" - - -class msg_ping_prebip31(object): - command = b"ping" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_ping() (pre-bip31)" - - -class msg_ping(object): - command = b"ping" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_ping(nonce=%08x)" % self.nonce - - -class msg_pong(object): - command = b"pong" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_pong(nonce=%08x)" % self.nonce - - -class msg_mempool(object): - command = b"mempool" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_mempool()" - -class msg_sendheaders(object): - command = b"sendheaders" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_sendheaders()" - - -# getheaders message has -# number of entries -# vector of hashes -# hash_stop (hash of last desired block header, 0 to get as many as possible) -class msg_getheaders(object): - command = b"getheaders" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getheaders(locator=%s, stop=%064x)" \ - % (repr(self.locator), self.hashstop) - - -# headers message has -# <count> <vector of block headers> -class msg_headers(object): - command = b"headers" - - def __init__(self, headers=None): - self.headers = headers if headers is not None else [] - - def deserialize(self, f): - # comment in bitcoind indicates these should be deserialized as blocks - blocks = deser_vector(f, CBlock) - for x in blocks: - self.headers.append(CBlockHeader(x)) - - def serialize(self): - blocks = [CBlock(x) for x in self.headers] - return ser_vector(blocks) - - def __repr__(self): - return "msg_headers(headers=%s)" % repr(self.headers) + def peer_connect(self, dstaddr, dstport, net="regtest"): + self.dstaddr = dstaddr + self.dstport = dstport + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sendbuf = b"" + self.recvbuf = b"" + self.state = "connecting" + self.network = net + self.disconnect = False + logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) -class msg_reject(object): - command = b"reject" - REJECT_MALFORMED = 1 + try: + self.connect((dstaddr, dstport)) + except: + self.handle_close() - def __init__(self): - self.message = b"" - self.code = 0 - self.reason = b"" - self.data = 0 - - def deserialize(self, f): - self.message = deser_string(f) - self.code = struct.unpack("<B", f.read(1))[0] - self.reason = deser_string(f) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - self.data = deser_uint256(f) - - def serialize(self): - r = ser_string(self.message) - r += struct.pack("<B", self.code) - r += ser_string(self.reason) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - r += ser_uint256(self.data) - return r - - def __repr__(self): - return "msg_reject: %s %d %s [%064x]" \ - % (self.message, self.code, self.reason, self.data) - -class msg_feefilter(object): - command = b"feefilter" - - def __init__(self, feerate=0): - self.feerate = feerate - - def deserialize(self, f): - self.feerate = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.feerate) - return r - - def __repr__(self): - return "msg_feefilter(feerate=%08x)" % self.feerate - -class msg_sendcmpct(object): - command = b"sendcmpct" + def peer_disconnect(self): + # Connection could have already been closed by other end. + if self.state == "connected": + self.disconnect_node() - def __init__(self): - self.announce = False - self.version = 1 + # Connection and disconnection methods - def deserialize(self, f): - self.announce = struct.unpack("<?", f.read(1))[0] - self.version = struct.unpack("<Q", f.read(8))[0] + def handle_connect(self): + """asyncore callback when a connection is opened.""" + if self.state != "connected": + logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) + self.state = "connected" + self.on_open() - def serialize(self): - r = b"" - r += struct.pack("<?", self.announce) - r += struct.pack("<Q", self.version) - return r + def handle_close(self): + """asyncore callback when a connection is closed.""" + logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) + self.state = "closed" + self.recvbuf = b"" + self.sendbuf = b"" + try: + self.close() + except: + pass + self.on_close() - def __repr__(self): - return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + def disconnect_node(self): + """Disconnect the p2p connection. -class msg_cmpctblock(object): - command = b"cmpctblock" + Called by the test logic thread. Causes the p2p connection + to be disconnected on the next iteration of the asyncore loop.""" + self.disconnect = True - def __init__(self, header_and_shortids = None): - self.header_and_shortids = header_and_shortids + # Socket read methods - def deserialize(self, f): - self.header_and_shortids = P2PHeaderAndShortIDs() - self.header_and_shortids.deserialize(f) + def handle_read(self): + """asyncore callback when data is read from the socket.""" + t = self.recv(8192) + if len(t) > 0: + self.recvbuf += t + self._on_data() - def serialize(self): - r = b"" - r += self.header_and_shortids.serialize() - return r + def _on_data(self): + """Try to read P2P messages from the recv buffer. - def __repr__(self): - return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + This method reads data from the buffer in a loop. It deserializes, + parses and verifies the P2P header, then passes the P2P payload to + the on_message callback for processing.""" + try: + while True: + if len(self.recvbuf) < 4: + return + if self.recvbuf[:4] != MAGIC_BYTES[self.network]: + raise ValueError("got garbage %s" % repr(self.recvbuf)) + if len(self.recvbuf) < 4 + 12 + 4 + 4: + return + command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] + msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] + checksum = self.recvbuf[4+12+4:4+12+4+4] + if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: + return + msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen] + th = sha256(msg) + h = sha256(th) + if checksum != h[:4]: + raise ValueError("got bad checksum " + repr(self.recvbuf)) + self.recvbuf = self.recvbuf[4+12+4+4+msglen:] + if command not in MESSAGEMAP: + raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + f = BytesIO(msg) + t = MESSAGEMAP[command]() + t.deserialize(f) + self._log_message("receive", t) + self.on_message(t) + except Exception as e: + logger.exception('Error reading message:', repr(e)) + raise -class msg_getblocktxn(object): - command = b"getblocktxn" + def on_message(self, message): + """Callback for processing a P2P payload. Must be overridden by derived class.""" + raise NotImplementedError - def __init__(self): - self.block_txn_request = None + # Socket write methods - def deserialize(self, f): - self.block_txn_request = BlockTransactionsRequest() - self.block_txn_request.deserialize(f) + def writable(self): + """asyncore method to determine whether the handle_write() callback should be called on the next loop.""" + with mininode_lock: + pre_connection = self.state == "connecting" + length = len(self.sendbuf) + return (length > 0 or pre_connection) - def serialize(self): - r = b"" - r += self.block_txn_request.serialize() - return r + def handle_write(self): + """asyncore callback when data should be written to the socket.""" + with mininode_lock: + # asyncore does not expose socket connection, only the first read/write + # event, thus we must check connection manually here to know when we + # actually connect + if self.state == "connecting": + self.handle_connect() + if not self.writable(): + return - def __repr__(self): - return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + try: + sent = self.send(self.sendbuf) + except: + self.handle_close() + return + self.sendbuf = self.sendbuf[sent:] -class msg_blocktxn(object): - command = b"blocktxn" + def send_message(self, message, pushbuf=False): + """Send a P2P message over the socket. - def __init__(self): - self.block_transactions = BlockTransactions() + This method takes a P2P payload, builds the P2P header and adds + the message to the send buffer to be sent over the socket.""" + if self.state != "connected" and not pushbuf: + raise IOError('Not connected, no pushbuf') + self._log_message("send", message) + command = message.command + data = message.serialize() + tmsg = MAGIC_BYTES[self.network] + tmsg += command + tmsg += b"\x00" * (12 - len(command)) + tmsg += struct.pack("<I", len(data)) + th = sha256(data) + h = sha256(th) + tmsg += h[:4] + tmsg += data + with mininode_lock: + if (len(self.sendbuf) == 0 and not pushbuf): + try: + sent = self.send(tmsg) + self.sendbuf = tmsg[sent:] + except BlockingIOError: + self.sendbuf = tmsg + else: + self.sendbuf += tmsg - def deserialize(self, f): - self.block_transactions.deserialize(f) + # Class utility methods - def serialize(self): - r = b"" - r += self.block_transactions.serialize() - return r + def _log_message(self, direction, msg): + """Logs a message being sent or received over the connection.""" + if direction == "send": + log_message = "Send message to " + elif direction == "receive": + log_message = "Received message from " + log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500]) + if len(log_message) > 500: + log_message += "... (msg truncated)" + logger.debug(log_message) - def __repr__(self): - return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) -class msg_witness_blocktxn(msg_blocktxn): - def serialize(self): - r = b"" - r += self.block_transactions.serialize(with_witness=True) - return r +class P2PInterface(P2PConnection): + """A high-level P2P interface class for communicating with a Bitcoin node. -class NodeConnCB(object): - """Callback and helper functions for P2P connection to a bitcoind node. + This class provides high-level callbacks for processing P2P message + payloads, as well as convenience methods for interacting with the + node over P2P. Individual testcases should subclass this and override the on_* methods - if they want to alter message handling behaviour. - """ - + if they want to alter message handling behaviour.""" def __init__(self): - # Track whether we have a P2P connection open to the node - self.connected = False - self.connection = None + super().__init__() # Track number of messages of each type received and the most recent # message of each type @@ -1474,100 +264,89 @@ class NodeConnCB(object): # A count of the number of ping messages we've sent to the node self.ping_counter = 1 - # deliver_sleep_time is helpful for debugging race conditions in p2p - # tests; it causes message delivery to sleep for the specified time - # before acquiring the global lock and delivering the next message. - self.deliver_sleep_time = None + # The network services received from the peer + self.nServices = 0 + + def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs): + super().peer_connect(*args, **kwargs) + + if send_version: + # Send a version msg + vt = msg_version() + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.send_message(vt, True) # Message receiving methods - def deliver(self, conn, message): + def on_message(self, message): """Receive message and dispatch message to appropriate callback. We keep a count of how many of each message type has been received - and the most recent message of each type. - - Optionally waits for deliver_sleep_time before dispatching message. - """ - - deliver_sleep = self.get_deliver_sleep_time() - if deliver_sleep is not None: - time.sleep(deliver_sleep) + and the most recent message of each type.""" with mininode_lock: try: command = message.command.decode('ascii') self.message_count[command] += 1 self.last_message[command] = message - getattr(self, 'on_' + command)(conn, message) + getattr(self, 'on_' + command)(message) except: - print("ERROR delivering %s (%s)" % (repr(message), - sys.exc_info()[0])) + print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise - def get_deliver_sleep_time(self): - with mininode_lock: - return self.deliver_sleep_time - # Callback methods. Can be overridden by subclasses in individual test # cases to provide custom message handling behaviour. - def on_open(self, conn): - self.connected = True - - def on_close(self, conn): - self.connected = False - self.connection = None - - def on_addr(self, conn, message): pass - def on_alert(self, conn, message): pass - def on_block(self, conn, message): pass - def on_blocktxn(self, conn, message): pass - def on_cmpctblock(self, conn, message): pass - def on_feefilter(self, conn, message): pass - def on_getaddr(self, conn, message): pass - def on_getblocks(self, conn, message): pass - def on_getblocktxn(self, conn, message): pass - def on_getdata(self, conn, message): pass - def on_getheaders(self, conn, message): pass - def on_headers(self, conn, message): pass - def on_mempool(self, conn): pass - def on_pong(self, conn, message): pass - def on_reject(self, conn, message): pass - def on_sendcmpct(self, conn, message): pass - def on_sendheaders(self, conn, message): pass - def on_tx(self, conn, message): pass - - def on_inv(self, conn, message): + def on_open(self): + pass + + def on_close(self): + pass + + def on_addr(self, message): pass + def on_block(self, message): pass + def on_blocktxn(self, message): pass + def on_cmpctblock(self, message): pass + def on_feefilter(self, message): pass + def on_getaddr(self, message): pass + def on_getblocks(self, message): pass + def on_getblocktxn(self, message): pass + def on_getdata(self, message): pass + def on_getheaders(self, message): pass + def on_headers(self, message): pass + def on_mempool(self, message): pass + def on_pong(self, message): pass + def on_reject(self, message): pass + def on_sendcmpct(self, message): pass + def on_sendheaders(self, message): pass + def on_tx(self, message): pass + + def on_inv(self, message): want = msg_getdata() for i in message.inv: if i.type != 0: want.inv.append(i) if len(want.inv): - conn.send_message(want) + self.send_message(want) - def on_ping(self, conn, message): - if conn.ver_send > BIP0031_VERSION: - conn.send_message(msg_pong(message.nonce)) + def on_ping(self, message): + self.send_message(msg_pong(message.nonce)) - def on_verack(self, conn, message): - conn.ver_recv = conn.ver_send + def on_verack(self, message): self.verack_received = True - def on_version(self, conn, message): - if message.nVersion >= 209: - conn.send_message(msg_verack()) - conn.ver_send = min(MY_VERSION, message.nVersion) - if message.nVersion < 209: - conn.ver_recv = conn.ver_send - conn.nServices = message.nServices + def on_version(self, message): + assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) + self.send_message(msg_verack()) + self.nServices = message.nServices # Connection helper methods - def add_connection(self, conn): - self.connection = conn - def wait_for_disconnect(self, timeout=60): - test_function = lambda: not self.connected + test_function = lambda: self.state != "connected" wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message receiving helper methods @@ -1599,12 +378,6 @@ class NodeConnCB(object): # Message sending helper functions - def send_message(self, message): - if self.connection: - self.connection.send_message(message) - else: - logger.error("Cannot send message. No connection to node!") - def send_and_ping(self, message): self.send_message(message) self.sync_with_ping() @@ -1615,218 +388,25 @@ class NodeConnCB(object): test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 - return True - -# The actual NodeConn class -# This class provides an interface for a p2p connection to a specified node -class NodeConn(asyncore.dispatcher): - messagemap = { - b"version": msg_version, - b"verack": msg_verack, - b"addr": msg_addr, - b"alert": msg_alert, - b"inv": msg_inv, - b"getdata": msg_getdata, - b"getblocks": msg_getblocks, - b"tx": msg_tx, - b"block": msg_block, - b"getaddr": msg_getaddr, - b"ping": msg_ping, - b"pong": msg_pong, - b"headers": msg_headers, - b"getheaders": msg_getheaders, - b"reject": msg_reject, - b"mempool": msg_mempool, - b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders, - b"sendcmpct": msg_sendcmpct, - b"cmpctblock": msg_cmpctblock, - b"getblocktxn": msg_getblocktxn, - b"blocktxn": msg_blocktxn - } - MAGIC_BYTES = { - "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet - "testnet3": b"\x0b\x11\x09\x07", # testnet3 - "regtest": b"\xfa\xbf\xb5\xda", # regtest - } - - def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True): - asyncore.dispatcher.__init__(self, map=mininode_socket_map) - self.dstaddr = dstaddr - self.dstport = dstport - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.sendbuf = b"" - self.recvbuf = b"" - self.ver_send = 209 - self.ver_recv = 209 - self.last_sent = 0 - self.state = "connecting" - self.network = net - self.cb = callback - self.disconnect = False - self.nServices = 0 - - if send_version: - # stuff version msg into sendbuf - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.send_message(vt, True) - logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) - try: - self.connect((dstaddr, dstport)) - except: - self.handle_close() - self.rpc = rpc - - def handle_connect(self): - if self.state != "connected": - logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "connected" - self.cb.on_open(self) - - def handle_close(self): - logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "closed" - self.recvbuf = b"" - self.sendbuf = b"" - try: - self.close() - except: - pass - self.cb.on_close(self) - - def handle_read(self): - t = self.recv(8192) - if len(t) > 0: - self.recvbuf += t - self.got_data() - - def readable(self): - return True - - def writable(self): - with mininode_lock: - pre_connection = self.state == "connecting" - length = len(self.sendbuf) - return (length > 0 or pre_connection) - - def handle_write(self): - with mininode_lock: - # asyncore does not expose socket connection, only the first read/write - # event, thus we must check connection manually here to know when we - # actually connect - if self.state == "connecting": - self.handle_connect() - if not self.writable(): - return - - try: - sent = self.send(self.sendbuf) - except: - self.handle_close() - return - self.sendbuf = self.sendbuf[sent:] - - def got_data(self): - try: - while True: - if len(self.recvbuf) < 4: - return - if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]: - raise ValueError("got garbage %s" % repr(self.recvbuf)) - if self.ver_recv < 209: - if len(self.recvbuf) < 4 + 12 + 4: - return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] - msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] - checksum = None - if len(self.recvbuf) < 4 + 12 + 4 + msglen: - return - msg = self.recvbuf[4+12+4:4+12+4+msglen] - self.recvbuf = self.recvbuf[4+12+4+msglen:] - else: - if len(self.recvbuf) < 4 + 12 + 4 + 4: - return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] - msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] - checksum = self.recvbuf[4+12+4:4+12+4+4] - if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: - return - msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen] - th = sha256(msg) - h = sha256(th) - if checksum != h[:4]: - raise ValueError("got bad checksum " + repr(self.recvbuf)) - self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command in self.messagemap: - f = BytesIO(msg) - t = self.messagemap[command]() - t.deserialize(f) - self.got_message(t) - else: - logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) - raise ValueError("Unknown command: '%s'" % (command)) - except Exception as e: - logger.exception('got_data:', repr(e)) - raise - - def send_message(self, message, pushbuf=False): - if self.state != "connected" and not pushbuf: - raise IOError('Not connected, no pushbuf') - self._log_message("send", message) - command = message.command - data = message.serialize() - tmsg = self.MAGIC_BYTES[self.network] - tmsg += command - tmsg += b"\x00" * (12 - len(command)) - tmsg += struct.pack("<I", len(data)) - if self.ver_send >= 209: - th = sha256(data) - h = sha256(th) - tmsg += h[:4] - tmsg += data - with mininode_lock: - if (len(self.sendbuf) == 0 and not pushbuf): - try: - sent = self.send(tmsg) - self.sendbuf = tmsg[sent:] - except BlockingIOError: - self.sendbuf = tmsg - else: - self.sendbuf += tmsg - self.last_sent = time.time() - - def got_message(self, message): - if message.command == b"version": - if message.nVersion <= BIP0031_VERSION: - self.messagemap[b'ping'] = msg_ping_prebip31 - if self.last_sent + 30 * 60 < time.time(): - self.send_message(self.messagemap[b'ping']()) - self._log_message("receive", message) - self.cb.deliver(self, message) - - def _log_message(self, direction, msg): - if direction == "send": - log_message = "Send message to " - elif direction == "receive": - log_message = "Received message from " - log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500]) - if len(log_message) > 500: - log_message += "... (msg truncated)" - logger.debug(log_message) +# Keep our own socket map for asyncore, so that we can track disconnects +# ourselves (to workaround an issue with closing an asyncore socket when +# using select) +mininode_socket_map = dict() - def disconnect_node(self): - self.disconnect = True +# One lock for synchronizing all data access between the networking thread (see +# NetworkThread below) and the thread running the test logic. For simplicity, +# P2PConnection acquires this lock whenever delivering a message to a P2PInterface, +# and whenever adding anything to the send buffer (in send_message()). This +# lock should be acquired in the thread running the test logic to synchronize +# access to any data shared with the P2PInterface or P2PConnection. +mininode_lock = threading.RLock() +class NetworkThread(threading.Thread): + def __init__(self): + super().__init__(name="NetworkThread") -class NetworkThread(Thread): def run(self): while mininode_socket_map: # We check for whether to disconnect outside of the asyncore @@ -1836,16 +416,27 @@ class NetworkThread(Thread): for fd, obj in mininode_socket_map.items(): if obj.disconnect: disconnected.append(obj) - [ obj.handle_close() for obj in disconnected ] + [obj.handle_close() for obj in disconnected] asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1) logger.debug("Network thread closing") +def network_thread_start(): + """Start the network thread.""" + # Only one network thread may run at a time + assert not network_thread_running() + + NetworkThread().start() + +def network_thread_running(): + """Return whether the network thread is running.""" + return any([thread.name == "NetworkThread" for thread in threading.enumerate()]) -# An exception we can raise if we detect a potential disconnect -# (p2p or rpc) before the test is complete -class EarlyDisconnectError(Exception): - def __init__(self, value): - self.value = value +def network_thread_join(timeout=10): + """Wait timeout seconds for the network thread to terminate. - def __str__(self): - return repr(self.value) + Throw if the network thread doesn't terminate in timeout seconds.""" + network_threads = [thread for thread in threading.enumerate() if thread.name == "NetworkThread"] + assert len(network_threads) <= 1 + for thread in network_threads: + thread.join(timeout) + assert not thread.is_alive() diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 8f5339a02a..a4c046bd3d 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -370,7 +370,7 @@ class CScriptTruncatedPushDataError(CScriptInvalidError): super(CScriptTruncatedPushDataError, self).__init__(msg) # This is used, eg, for blockchain heights in coinbase scripts (bip34) -class CScriptNum(object): +class CScriptNum(): def __init__(self, d=0): self.value = d diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index 0070844168..7b40c47fbf 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -31,7 +31,7 @@ def recvall(s, n): return rv ### Implementation classes -class Socks5Configuration(object): +class Socks5Configuration(): """Proxy configuration.""" def __init__(self): self.addr = None # Bind address (must be set) @@ -39,7 +39,7 @@ class Socks5Configuration(object): self.unauth = False # Support unauthenticated self.auth = False # Support authentication -class Socks5Command(object): +class Socks5Command(): """Information about an incoming socks5 command.""" def __init__(self, cmd, atyp, addr, port, username, password): self.cmd = cmd # Command (one of Command.*) @@ -51,7 +51,7 @@ class Socks5Command(object): def __repr__(self): return 'Socks5Command(%s,%s,%s,%s,%s,%s)' % (self.cmd, self.atyp, self.addr, self.port, self.username, self.password) -class Socks5Connection(object): +class Socks5Connection(): def __init__(self, serv, conn, peer): self.serv = serv self.conn = conn @@ -122,7 +122,7 @@ class Socks5Connection(object): finally: self.conn.close() -class Socks5Server(object): +class Socks5Server(): def __init__(self, conf): self.conf = conf self.s = socket.socket(conf.af) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 381513ab9e..a46312d62c 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" -from collections import deque from enum import Enum import logging import optparse @@ -14,7 +13,6 @@ import shutil import sys import tempfile import time -import traceback from .authproxy import JSONRPCException from . import coverage @@ -43,7 +41,7 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 -class BitcoinTestFramework(object): +class BitcoinTestFramework(): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. @@ -102,8 +100,11 @@ class BitcoinTestFramework(object): check_json_precision() + self.options.cachedir = os.path.abspath(self.options.cachedir) + # Set up temp directory and start logging if self.options.tmpdir: + self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") @@ -146,32 +147,19 @@ class BitcoinTestFramework(object): shutil.rmtree(self.options.tmpdir) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) - if os.getenv("PYTHON_DEBUG", ""): - # Dump the end of the debug logs, to aid in debugging rare - # travis failures. - import glob - filenames = [self.options.tmpdir + "/test_framework.log"] - filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") - MAX_LINES_TO_PRINT = 1000 - for fn in filenames: - try: - with open(fn, 'r') as f: - print("From", fn, ":") - print("".join(deque(f, MAX_LINES_TO_PRINT))) - except OSError: - print("Opening file %s failed." % fn) - traceback.print_exc() if success == TestStatus.PASSED: self.log.info("Tests successful") - sys.exit(TEST_EXIT_PASSED) + exit_code = TEST_EXIT_PASSED elif success == TestStatus.SKIPPED: self.log.info("Test skipped") - sys.exit(TEST_EXIT_SKIPPED) + exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) - logging.shutdown() - sys.exit(TEST_EXIT_FAILED) + self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) + exit_code = TEST_EXIT_FAILED + logging.shutdown() + sys.exit(exit_code) # Methods to override in subclass test scripts. def set_test_params(self): @@ -429,7 +417,7 @@ class BitcoinTestFramework(object): self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(self.options.cachedir, i, "debug.log")) - os.remove(log_filename(self.options.cachedir, i, "db.log")) + os.remove(log_filename(self.options.cachedir, i, "wallets/db.log")) os.remove(log_filename(self.options.cachedir, i, "peers.dat")) os.remove(log_filename(self.options.cachedir, i, "fee_estimates.dat")) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 12dab57a02..a9248c764e 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -13,13 +13,14 @@ import os import subprocess import time +from .authproxy import JSONRPCException from .util import ( assert_equal, get_rpc_proxy, rpc_url, wait_until, + p2p_port, ) -from .authproxy import JSONRPCException BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -31,9 +32,11 @@ class TestNode(): - state about the node (whether it's running, etc) - a Python subprocess.Popen object representing the running process - an RPC connection to the node + - one or more P2P connections to the node + - To make things easier for the test writer, a bit of magic is happening under the covers. - Any unrecognised messages will be dispatched to the RPC connection.""" + To make things easier for the test writer, any unrecognised messages will + be dispatched to the RPC connection.""" def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): self.index = i @@ -63,10 +66,12 @@ class TestNode(): self.url = None self.log = logging.getLogger('TestFramework.node%d' % i) - def __getattr__(self, *args, **kwargs): + self.p2ps = [] + + def __getattr__(self, name): """Dispatches any unrecognised messages to the RPC connection.""" assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" - return self.rpc.__getattr__(*args, **kwargs) + return getattr(self.rpc, name) def start(self, extra_args=None, stderr=None): """Start the node.""" @@ -119,6 +124,7 @@ class TestNode(): self.stop() except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") + del self.p2ps[:] def is_node_stopped(self): """Checks whether the node has stopped. @@ -151,6 +157,37 @@ class TestNode(): self.encryptwallet(passphrase) self.wait_until_stopped() + def add_p2p_connection(self, p2p_conn, *args, **kwargs): + """Add a p2p connection to the node. + + This method adds the p2p connection to the self.p2ps list and also + returns the connection to the caller.""" + if 'dstport' not in kwargs: + kwargs['dstport'] = p2p_port(self.index) + if 'dstaddr' not in kwargs: + kwargs['dstaddr'] = '127.0.0.1' + + p2p_conn.peer_connect(*args, **kwargs) + self.p2ps.append(p2p_conn) + + return p2p_conn + + @property + def p2p(self): + """Return the first p2p connection + + Convenience property - most tests only use a single p2p connection to each + node, so this saves having to write node.p2ps[0] many times.""" + assert self.p2ps, "No p2p connection" + return self.p2ps[0] + + def disconnect_p2ps(self): + """Close all p2p connections to the node.""" + for p in self.p2ps: + p.peer_disconnect() + del self.p2ps[:] + + class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 8c4651f6e0..bfd1192d3a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -15,6 +15,7 @@ For a description of arguments recognized by test scripts, see """ import argparse +from collections import deque import configparser import datetime import os @@ -124,6 +125,11 @@ BASE_SCRIPTS= [ 'resendwallettransactions.py', 'minchainwork.py', 'p2p-fingerprint.py', + 'uacomment.py', + 'p2p-acceptblock.py', + 'feature_logging.py', + 'node_network_limited.py', + 'conf_args.py', ] EXTENDED_SCRIPTS = [ @@ -151,7 +157,6 @@ EXTENDED_SCRIPTS = [ 'txn_clone.py --mineblock', 'notifications.py', 'invalidateblock.py', - 'p2p-acceptblock.py', 'replace-by-fee.py', ] @@ -173,6 +178,7 @@ def main(): epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--combinedlogslen', '-c', type=int, default=0, help='print a combined log (of length n lines) from all test nodes and test framework to the console on failure.') parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') @@ -265,9 +271,9 @@ def main(): if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.combinedlogslen) -def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[], combined_logs_len=0): # Warn if bitcoind is already running (unix only) try: if subprocess.check_output(["pidof", "bitcoind"]) is not None: @@ -299,7 +305,11 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + try: + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + except Exception as e: + print(e.output) + raise e #Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) @@ -309,7 +319,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove max_len_name = len(max(test_list, key=len)) for _ in range(len(test_list)): - test_result, stdout, stderr = job_queue.get_next() + test_result, testdir, stdout, stderr = job_queue.get_next() test_results.append(test_result) if test_result.status == "Passed": @@ -320,6 +330,14 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') + if combined_logs_len and os.path.isdir(testdir): + # Print the final `combinedlogslen` lines of the combined logs + print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0])) + print('\n============') + print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) + print('============\n') + combined_logs, _ = subprocess.Popen([os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate() + print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) print_results(test_results, max_len_name, (int(time.time() - time0))) @@ -384,13 +402,15 @@ class TestHandler: log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) test_argv = t.split() - tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] + testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed) + tmpdir_arg = ["--tmpdir={}".format(testdir)] self.jobs.append((t, time.time(), - subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, + subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), + testdir, log_stdout, log_stderr)) if not self.jobs: @@ -399,7 +419,7 @@ class TestHandler: # Return first proc that finishes time.sleep(.5) for j in self.jobs: - (name, time0, proc, log_out, log_err) = j + (name, time0, proc, testdir, log_out, log_err) = j if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60: # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not # providing useful output. @@ -417,7 +437,7 @@ class TestHandler: self.num_running -= 1 self.jobs.remove(j) - return TestResult(name, status, int(time.time() - time0)), stdout, stderr + return TestResult(name, status, int(time.time() - time0)), testdir, stdout, stderr print('.', end='', flush=True) class TestResult(): @@ -459,7 +479,7 @@ def check_script_list(src_dir): # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) -class RPCCoverage(object): +class RPCCoverage(): """ Coverage reporting utilities for test_runner. diff --git a/test/functional/uacomment.py b/test/functional/uacomment.py new file mode 100755 index 0000000000..0b2c64ab69 --- /dev/null +++ b/test/functional/uacomment.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the -uacomment option.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class UacommentTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self.log.info("test multiple -uacomment") + test_uacomment = self.nodes[0].getnetworkinfo()["subversion"][-12:-1] + assert_equal(test_uacomment, "(testnode0)") + + self.restart_node(0, ["-uacomment=foo"]) + foo_uacomment = self.nodes[0].getnetworkinfo()["subversion"][-17:-1] + assert_equal(foo_uacomment, "(testnode0; foo)") + + self.log.info("test -uacomment max length") + self.stop_node(0) + expected = "Total length of network version string (286) exceeds maximum length (256). Reduce the number or size of uacomments." + self.assert_start_raises_init_error(0, ["-uacomment=" + 'a' * 256], expected) + + self.log.info("test -uacomment unsafe characters") + for unsafe_char in ['/', ':', '(', ')']: + expected = "User Agent comment (" + unsafe_char + ") contains unsafe characters" + self.assert_start_raises_init_error(0, ["-uacomment=" + unsafe_char], expected) + +if __name__ == '__main__': + UacommentTest().main() diff --git a/test/functional/wallet-accounts.py b/test/functional/wallet-accounts.py index 40726d2a76..bc1efaee15 100755 --- a/test/functional/wallet-accounts.py +++ b/test/functional/wallet-accounts.py @@ -72,62 +72,135 @@ class WalletAccountsTest(BitcoinTestFramework): # otherwise we're off by exactly the fee amount as that's mined # and matures in the next 100 blocks node.sendfrom("", common_address, fee) - accounts = ["a", "b", "c", "d", "e"] amount_to_send = 1.0 - account_addresses = dict() + + # Create accounts and make sure subsequent account API calls + # recognize the account/address associations. + accounts = [Account(name) for name in ("a", "b", "c", "d", "e")] for account in accounts: - address = node.getaccountaddress(account) - account_addresses[account] = address - - node.getnewaddress(account) - assert_equal(node.getaccount(address), account) - assert(address in node.getaddressesbyaccount(account)) - - node.sendfrom("", address, amount_to_send) - + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + + # Send a transaction to each account, and make sure this forces + # getaccountaddress to generate a new receiving address. + for account in accounts: + node.sendtoaddress(account.receive_address, amount_to_send) + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + + # Check the amounts received. node.generate(1) + for account in accounts: + assert_equal( + node.getreceivedbyaddress(account.addresses[0]), amount_to_send) + assert_equal(node.getreceivedbyaccount(account.name), amount_to_send) - for i in range(len(accounts)): - from_account = accounts[i] + # Check that sendfrom account reduces listaccounts balances. + for i, account in enumerate(accounts): to_account = accounts[(i+1) % len(accounts)] - to_address = account_addresses[to_account] - node.sendfrom(from_account, to_address, amount_to_send) - + node.sendfrom(account.name, to_account.receive_address, amount_to_send) node.generate(1) - for account in accounts: - address = node.getaccountaddress(account) - assert(address != account_addresses[account]) - assert_equal(node.getreceivedbyaccount(account), 2) - node.move(account, "", node.getbalance(account)) - + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + assert_equal(node.getreceivedbyaccount(account.name), 2) + node.move(account.name, "", node.getbalance(account.name)) + account.verify(node) node.generate(101) - expected_account_balances = {"": 5200} for account in accounts: - expected_account_balances[account] = 0 - + expected_account_balances[account.name] = 0 assert_equal(node.listaccounts(), expected_account_balances) - assert_equal(node.getbalance(""), 5200) + # Check that setaccount can assign an account to a new unused address. for account in accounts: address = node.getaccountaddress("") - node.setaccount(address, account) - assert(address in node.getaddressesbyaccount(account)) + node.setaccount(address, account.name) + account.add_address(address) + account.verify(node) assert(address not in node.getaddressesbyaccount("")) + # Check that addmultisigaddress can assign accounts. for account in accounts: addresses = [] for x in range(10): addresses.append(node.getnewaddress()) - multisig_address = node.addmultisigaddress(5, addresses, account) + multisig_address = node.addmultisigaddress(5, addresses, account.name) + account.add_address(multisig_address) + account.verify(node) node.sendfrom("", multisig_address, 50) - node.generate(101) - for account in accounts: - assert_equal(node.getbalance(account), 50) + assert_equal(node.getbalance(account.name), 50) + + # Check that setaccount can change the account of an address from a + # different account. + change_account(node, accounts[0].addresses[0], accounts[0], accounts[1]) + + # Check that setaccount can change the account of an address which + # is the receiving address of a different account. + change_account(node, accounts[0].receive_address, accounts[0], accounts[1]) + + # Check that setaccount can set the account of an address already + # in the account. This is a no-op. + change_account(node, accounts[2].addresses[0], accounts[2], accounts[2]) + + # Check that setaccount can set the account of an address which is + # already the receiving address of the account. It would probably make + # sense for this to be a no-op, but right now it resets the receiving + # address, causing getaccountaddress to return a brand new address. + change_account(node, accounts[2].receive_address, accounts[2], accounts[2]) + +class Account: + def __init__(self, name): + # Account name + self.name = name + # Current receiving address associated with this account. + self.receive_address = None + # List of all addresses assigned with this account + self.addresses = [] + + def add_address(self, address): + assert_equal(address not in self.addresses, True) + self.addresses.append(address) + + def add_receive_address(self, address): + self.add_address(address) + self.receive_address = address + + def verify(self, node): + if self.receive_address is not None: + assert self.receive_address in self.addresses + assert_equal(node.getaccountaddress(self.name), self.receive_address) + + for address in self.addresses: + assert_equal(node.getaccount(address), self.name) + + assert_equal( + set(node.getaddressesbyaccount(self.name)), set(self.addresses)) + + +def change_account(node, address, old_account, new_account): + assert_equal(address in old_account.addresses, True) + node.setaccount(address, new_account.name) + + old_account.addresses.remove(address) + new_account.add_address(address) + + # Calling setaccount on an address which was previously the receiving + # address of a different account should reset the receiving address of + # the old account, causing getaccountaddress to return a brand new + # address. + if address == old_account.receive_address: + new_address = node.getaccountaddress(old_account.name) + assert_equal(new_address not in old_account.addresses, True) + assert_equal(new_address not in new_account.addresses, True) + old_account.add_receive_address(new_address) + + old_account.verify(node) + new_account.verify(node) + if __name__ == '__main__': WalletAccountsTest().main() diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 47de8777a6..85812f9acc 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -10,13 +10,14 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import (assert_equal, assert_raises_rpc_error) -def read_dump(file_name, addrs, hd_master_addr_old): +def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): """ Read the given dump, count the addrs that match, count change and reserve. Also check that the old hd_master is inactive """ with open(file_name, encoding='utf8') as inputfile: found_addr = 0 + found_script_addr = 0 found_addr_chg = 0 found_addr_rsv = 0 hd_master_addr_ret = None @@ -38,6 +39,9 @@ def read_dump(file_name, addrs, hd_master_addr_old): # ensure we have generated a new hd master key assert(hd_master_addr_old != addr) hd_master_addr_ret = addr + elif keytype == "script=1": + # scripts don't have keypaths + keypath = None else: keypath = addr_keypath.rstrip().split("hdkeypath=")[1] @@ -52,7 +56,14 @@ def read_dump(file_name, addrs, hd_master_addr_old): elif keytype == "reserve=1": found_addr_rsv += 1 break - return found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret + + # count scripts + for script_addr in script_addrs: + if script_addr == addr.rstrip() and keytype == "script=1": + found_script_addr += 1 + break + + return found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): @@ -81,13 +92,19 @@ class WalletDumpTest(BitcoinTestFramework): # Should be a no-op: self.nodes[0].keypoolrefill() + # Test scripts dump by adding a P2SH witness and a 1-of-1 multisig address + witness_addr = self.nodes[0].addwitnessaddress(addrs[0]["address"], True) + multisig_addr = self.nodes[0].addmultisigaddress(1, [addrs[1]["address"]]) + script_addrs = [witness_addr, multisig_addr] + # dump unencrypted wallet result = self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") assert_equal(result['filename'], os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) - found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ - read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) + found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, script_addrs, None) assert_equal(found_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_script_addr, 2) # all scripts must be in the dump assert_equal(found_addr_chg, 50) # 50 blocks where mined assert_equal(found_addr_rsv, 90*2) # 90 keys plus 100% internal keys @@ -99,14 +116,29 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump") - found_addr, found_addr_chg, found_addr_rsv, _ = \ - read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) + found_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ + read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, script_addrs, hd_master_addr_unenc) assert_equal(found_addr, test_addr_count) + assert_equal(found_script_addr, 2) assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now assert_equal(found_addr_rsv, 90*2) # Overwriting should fail assert_raises_rpc_error(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") + # Restart node with new wallet, and test importwallet + self.stop_node(0) + self.start_node(0, ['-wallet=w2']) + + # Make sure the address is not IsMine before import + result = self.nodes[0].validateaddress(multisig_addr) + assert(result['ismine'] == False) + + self.nodes[0].importwallet(os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) + + # Now check IsMine is true + result = self.nodes[0].validateaddress(multisig_addr) + assert(result['ismine'] == True) + if __name__ == '__main__': WalletDumpTest().main () diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index 5ef3bf5bff..d21656a971 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -10,6 +10,7 @@ from test_framework.util import ( connect_nodes_bi, ) import shutil +import os class WalletHDTest(BitcoinTestFramework): def set_test_params(self): @@ -70,9 +71,9 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) # we need to delete the complete regtest directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(tmpdir + "/node1/regtest/blocks") - shutil.rmtree(tmpdir + "/node1/regtest/chainstate") - shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) + shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallets/wallet.dat")) self.start_node(1) # Assert that derivation is deterministic @@ -91,6 +92,22 @@ class WalletHDTest(BitcoinTestFramework): self.start_node(1, extra_args=self.extra_args[1] + ['-rescan']) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) + # Try a RPC based rescan + self.stop_node(1) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) + shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallet.dat")) + self.start_node(1, extra_args=self.extra_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + self.sync_all() + out = self.nodes[1].rescanblockchain(0, 1) + assert_equal(out['start_height'], 0) + assert_equal(out['stop_height'], 1) + out = self.nodes[1].rescanblockchain() + assert_equal(out['start_height'], 0) + assert_equal(out['stop_height'], self.nodes[1].getblockcount()) + assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) + # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] diff --git a/test/functional/wallet.py b/test/functional/wallet.py index d0ca24119c..e0da5ce136 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -103,11 +103,19 @@ class WalletTest(BitcoinTestFramework): # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} + assert_raises_rpc_error(-8, "Invalid parameter, expected locked output", self.nodes[2].lockunspent, True, [unspent_0]) self.nodes[2].lockunspent(False, [unspent_0]) + assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) + assert_raises_rpc_error(-8, "Invalid parameter, unknown transaction", + self.nodes[2].lockunspent, False, + [{"txid": "0000000000000000000000000000000000", "vout": 0}]) + assert_raises_rpc_error(-8, "Invalid parameter, vout index out of bounds", + self.nodes[2].lockunspent, False, + [{"txid": unspent_0["txid"], "vout": 999}]) # Have node1 generate 100 blocks (so node0 can recover the fee) self.nodes[1].generate(100) @@ -146,6 +154,10 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 94) assert_equal(self.nodes[2].getbalance("from1"), 94-21) + # Verify that a spent output cannot be locked anymore + spent_0 = {"txid": node0utxos[0]["txid"], "vout": node0utxos[0]["vout"]} + assert_raises_rpc_error(-8, "Invalid parameter, expected unspent output", self.nodes[0].lockunspent, False, [spent_0]) + # Send 10 BTC normal address = self.nodes[0].getnewaddress("test") fee_per_byte = Decimal('0.001') / 1000 diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 15ea26afa1..8ef5620cd8 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -90,9 +90,9 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node2/regtest/wallet.dat") + os.remove(self.options.tmpdir + "/node0/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node1/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node2/regtest/wallets/wallet.dat") def run_test(self): self.log.info("Generating initial blockchain") @@ -154,9 +154,9 @@ class WalletBackupTest(BitcoinTestFramework): shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") # Restore wallets from backup - shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallets/wallet.dat") self.log.info("Re-starting nodes") self.start_three() @@ -190,6 +190,16 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) + # Backup to source wallet file must fail + sourcePaths = [ + tmpdir + "/node0/regtest/wallets/wallet.dat", + tmpdir + "/node0/./regtest/wallets/wallet.dat", + tmpdir + "/node0/regtest/wallets/", + tmpdir + "/node0/regtest/wallets"] + + for sourcePath in sourcePaths: + assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + if __name__ == '__main__': WalletBackupTest().main() diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index 382ef5bae2..fa30318416 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -2,16 +2,37 @@ # Copyright (c) 2015-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the ZMQ API.""" +"""Test the ZMQ notification interface.""" import configparser import os import struct from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.mininode import CTransaction from test_framework.util import (assert_equal, bytes_to_hex_str, hash256, ) +from io import BytesIO + +class ZMQSubscriber: + def __init__(self, socket, topic): + self.sequence = 0 + self.socket = socket + self.topic = topic + + import zmq + self.socket.setsockopt(zmq.SUBSCRIBE, self.topic) + + def receive(self): + topic, body, seq = self.socket.recv_multipart() + # Topic should match the subscriber topic. + assert_equal(topic, self.topic) + # Sequence should be incremental. + assert_equal(struct.unpack('<I', seq)[-1], self.sequence) + self.sequence += 1 + return body + class ZMQTest (BitcoinTestFramework): def set_test_params(self): @@ -24,26 +45,33 @@ class ZMQTest (BitcoinTestFramework): except ImportError: raise SkipTest("python3-zmq module not available.") - # Check that bitcoin has been built with ZMQ enabled + # Check that bitcoin has been built with ZMQ enabled. config = configparser.ConfigParser() if not self.options.configfile: - self.options.configfile = os.path.dirname(__file__) + "/../config.ini" + self.options.configfile = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config.ini")) config.read_file(open(self.options.configfile)) if not config["components"].getboolean("ENABLE_ZMQ"): raise SkipTest("bitcoind has not been built with zmq enabled.") - self.zmqContext = zmq.Context() - self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.set(zmq.RCVTIMEO, 60000) - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"rawblock") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"rawtx") - ip_address = "tcp://127.0.0.1:28332" - self.zmqSubSocket.connect(ip_address) - self.extra_args = [['-zmqpubhashblock=%s' % ip_address, '-zmqpubhashtx=%s' % ip_address, - '-zmqpubrawblock=%s' % ip_address, '-zmqpubrawtx=%s' % ip_address], []] + # Initialize ZMQ context and socket. + # All messages are received in the same socket which means + # that this test fails if the publishing order changes. + # Note that the publishing order is not defined in the documentation and + # is subject to change. + address = "tcp://127.0.0.1:28332" + self.zmq_context = zmq.Context() + socket = self.zmq_context.socket(zmq.SUB) + socket.set(zmq.RCVTIMEO, 60000) + socket.connect(address) + + # Subscribe to all available topics. + self.hashblock = ZMQSubscriber(socket, b"hashblock") + self.hashtx = ZMQSubscriber(socket, b"hashtx") + self.rawblock = ZMQSubscriber(socket, b"rawblock") + self.rawtx = ZMQSubscriber(socket, b"rawtx") + + self.extra_args = [["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [self.hashblock, self.hashtx, self.rawblock, self.rawtx]], []] self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() @@ -51,103 +79,48 @@ class ZMQTest (BitcoinTestFramework): try: self._zmq_test() finally: - # Destroy the zmq context - self.log.debug("Destroying zmq context") - self.zmqContext.destroy(linger=None) + # Destroy the ZMQ context. + self.log.debug("Destroying ZMQ context") + self.zmq_context.destroy(linger=None) def _zmq_test(self): - genhashes = self.nodes[0].generate(1) + num_blocks = 5 + self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) + genhashes = self.nodes[0].generate(num_blocks) self.sync_all() - self.log.info("Wait for tx") - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashtx") - txhash = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on hashtx - - # rawtx - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawtx") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on rawtx - - # Check that the rawtx hashes to the hashtx - assert_equal(hash256(body), txhash) - - self.log.info("Wait for block") - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashblock") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on hashblock - blkhash = bytes_to_hex_str(body) - assert_equal(genhashes[0], blkhash) # blockhash from generate must be equal to the hash received over zmq - - # rawblock - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawblock") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) #must be sequence 0 on rawblock - - # Check the hash of the rawblock's header matches generate - assert_equal(genhashes[0], bytes_to_hex_str(hash256(body[:80]))) - - self.log.info("Generate 10 blocks (and 10 coinbase txes)") - n = 10 - genhashes = self.nodes[1].generate(n) - self.sync_all() + for x in range(num_blocks): + # Should receive the coinbase txid. + txid = self.hashtx.receive() + + # Should receive the coinbase raw transaction. + hex = self.rawtx.receive() + tx = CTransaction() + tx.deserialize(BytesIO(hex)) + tx.calc_sha256() + assert_equal(tx.hash, bytes_to_hex_str(txid)) - zmqHashes = [] - zmqRawHashed = [] - blockcount = 0 - for x in range(n * 4): - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - body = msg[1] - if topic == b"hashblock": - zmqHashes.append(bytes_to_hex_str(body)) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount + 1) - blockcount += 1 - if topic == b"rawblock": - zmqRawHashed.append(bytes_to_hex_str(hash256(body[:80]))) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount) - - for x in range(n): - assert_equal(genhashes[x], zmqHashes[x]) # blockhash from generate must be equal to the hash received over zmq - assert_equal(genhashes[x], zmqRawHashed[x]) + # Should receive the generated block hash. + hash = bytes_to_hex_str(self.hashblock.receive()) + assert_equal(genhashes[x], hash) + # The block should only have the coinbase txid. + assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + + # Should receive the generated raw block. + block = self.rawblock.receive() + assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) self.log.info("Wait for tx from second node") - # test tx from a second node - hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + payment_txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) self.sync_all() - # now we should receive a zmq msg because the tx was broadcast - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashtx") - body = msg[1] - hashZMQ = bytes_to_hex_str(body) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount + 1) - - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawtx") - body = msg[1] - hashedZMQ = bytes_to_hex_str(hash256(body)) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount+1) - assert_equal(hashRPC, hashZMQ) # txid from sendtoaddress must be equal to the hash received over zmq - assert_equal(hashRPC, hashedZMQ) + # Should receive the broadcasted txid. + txid = self.hashtx.receive() + assert_equal(payment_txid, bytes_to_hex_str(txid)) + + # Should receive the broadcasted raw transaction. + hex = self.rawtx.receive() + assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) if __name__ == '__main__': ZMQTest().main() |